R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Improve consistency throughout packedstore class

Browse Source 
No modifications to the existing logic have been performed. Some parameter names have been renamed to improve code readability and maintainability.
This commit is contained in:
Kawe Mazidjatari 2022-11-16 00:54:51 +01:00
parent 4b1c5dd5a3
commit 34d8ea578a
3 changed files with 151 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

228
r5dev/vpklib/packedstore.cpp
View File

@ -5,9 +5,12 @@
//=============================================================================// //=============================================================================//
// packedstore.cpp // packedstore.cpp
// //
// Note: VPK's are created in pairs of a directory file and block archive(s). // Note: VPK's are created in pairs of a directory file and pack file(s).
// - <locale><context>_<source>.bsp.pak000_dir.vpk --> directory file. // - <locale><target>_<level>.bsp.pak000_dir.vpk --> directory file.
// - <context>_<source>.bsp.pak000_<patch>.vpk --> block archive. // - <target>_<level>.bsp.pak000_<patch>.vpk --> pack file.
//
// - Assets larger than 1MiB are fragmented into chunks of 1MiB or smaller (ENTRY_MAX_LEN).
// - A VPK directory file could be patched up to 512 times before full rebuild is required.
// //
///////////////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////////////////
#include "core/stdafx.h" #include "core/stdafx.h"
@ -81,12 +84,12 @@ VPKDir_t CPackedStore::GetDirectoryFile(const string& svPackDirFile, bool bSanit
string svPackDirPrefix; string svPackDirPrefix;
svPackDirPrefix.append(DIR_LOCALE[0]); svPackDirPrefix.append(DIR_LOCALE[0]);
for (const string& svContext : DIR_CONTEXT) for (const string& svTarget : DIR_TARGET)
{ {
if (svSanitizedName.find(svContext) != string::npos) if (svSanitizedName.find(svTarget) != string::npos)
{ {
svPackDirPrefix.append(svContext); svPackDirPrefix.append(svTarget);
StringReplace(svSanitizedName, svContext, svPackDirPrefix); StringReplace(svSanitizedName, svTarget, svPackDirPrefix);
break; break;
} }
@ -99,15 +102,15 @@ VPKDir_t CPackedStore::GetDirectoryFile(const string& svPackDirFile, bool bSanit
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: formats pack file path for specific directory file // Purpose: formats pack file path for specific directory file
// Input : &svPackDirFile - // Input : &svPackDirFile -
// iArchiveIndex - // iPackFileIndex -
// output : string // output : string
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
string CPackedStore::GetPackFile(const string& svPackDirFile, uint16_t iArchiveIndex) const string CPackedStore::GetPackFile(const string& svPackDirFile, uint16_t iPackFileIndex) const
{ {
string svPackChunkFile = StripLocalePrefix(svPackDirFile); string svPackChunkFile = StripLocalePrefix(svPackDirFile);
ostringstream oss; ostringstream oss;
oss << std::setw(3) << std::setfill('0') << iArchiveIndex; oss << std::setw(3) << std::setfill('0') << iPackFileIndex;
string svPackChunkIndex = "pak000_" + oss.str(); string svPackChunkIndex = "pak000_" + oss.str();
StringReplace(svPackChunkFile, "pak000_dir", svPackChunkIndex); StringReplace(svPackChunkFile, "pak000_dir", svPackChunkIndex);
@ -241,7 +244,7 @@ vector<string> CPackedStore::GetEntryPaths(const string& svPathIn, const nlohman
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: gets the parts of the directory file name // Purpose: gets the parts of the directory file name
// Input : &svDirectoryName - // Input : &svDirectoryName -
// nCaptureGroup - (1 = locale + context, 2 = levelname) // nCaptureGroup - (1 = locale + target, 2 = level)
// Output : part of directory file name as string // Output : part of directory file name as string
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
string CPackedStore::GetNameParts(const string& svDirectoryName, int nCaptureGroup) const string CPackedStore::GetNameParts(const string& svDirectoryName, int nCaptureGroup) const
@ -253,11 +256,11 @@ string CPackedStore::GetNameParts(const string& svDirectoryName, int nCaptureGro
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: gets the source of the directory file name // Purpose: gets the level name from the directory file name
// Input : &svDirectoryName - // Input : &svDirectoryName -
// Output : source name as string (e.g. "mp_rr_box") // Output : level name as string (e.g. "mp_rr_box")
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
string CPackedStore::GetSourceName(const string& svDirectoryName) const string CPackedStore::GetLevelName(const string& svDirectoryName) const
{ {
std::smatch smRegexMatches; std::smatch smRegexMatches;
std::regex_search(svDirectoryName, smRegexMatches, DIR_REGEX); std::regex_search(svDirectoryName, smRegexMatches, DIR_REGEX);
@ -267,14 +270,14 @@ string CPackedStore::GetSourceName(const string& svDirectoryName) const
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: gets the manifest file associated with the VPK name // Purpose: gets the manifest file associated with the VPK name
// Input : &svWorkSpace - // Input : &svWorkspace -
// &svManifestName - // &svManifestName -
// Output : json // Output : parsed manifest as json
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
nlohmann::json CPackedStore::GetManifest(const string& svWorkSpace, const string& svManifestName) const nlohmann::json CPackedStore::GetManifest(const string& svWorkspace, const string& svManifestName) const
{ {
ostringstream ostream; ostringstream ostream;
ostream << svWorkSpace << "manifest/" << svManifestName << ".json"; ostream << svWorkspace << "manifest/" << svManifestName << ".json";
fs::path fsPath = fs::current_path() /= ostream.str(); fs::path fsPath = fs::current_path() /= ostream.str();
nlohmann::json jsOut; nlohmann::json jsOut;
@ -298,12 +301,12 @@ nlohmann::json CPackedStore::GetManifest(const string& svWorkSpace, const string
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: gets the contents from the global ignore list (.vpkignore) // Purpose: gets the contents from the global ignore list (.vpkignore)
// Input : &svWorkSpace - // Input : &svWorkspace -
// Output : a string vector of ignored directories/files // Output : a string vector of ignored directories/files and extensions
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
vector<string> CPackedStore::GetIgnoreList(const string& svWorkSpace) const vector<string> CPackedStore::GetIgnoreList(const string& svWorkspace) const
{ {
fs::path fsIgnore = svWorkSpace + ".vpkignore"; fs::path fsIgnore = svWorkspace + ".vpkignore";
ifstream iStream(fsIgnore); ifstream iStream(fsIgnore);
vector<string> vIgnore; vector<string> vIgnore;
@ -366,25 +369,25 @@ string CPackedStore::StripLocalePrefix(const string& svDirectoryFile) const
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: builds a valid file name for the VPK // Purpose: builds a valid file name for the VPK
// Input : svLanguage - // Input : svLanguage -
// svContext - // svTarget -
// &svPakName - // &svLevel -
// nPatch - // nPatch -
// Output : a vpk file pair (block and directory file names) // Output : a vpk file pair (block and directory file names)
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKPair_t CPackedStore::BuildFileName(string svLanguage, string svContext, const string& svPakName, int nPatch) const VPKPair_t CPackedStore::BuildFileName(string svLanguage, string svTarget, const string& svLevel, int nPatch) const
{ {
if (std::find(DIR_LOCALE.begin(), DIR_LOCALE.end(), svLanguage) == DIR_LOCALE.end()) if (std::find(DIR_LOCALE.begin(), DIR_LOCALE.end(), svLanguage) == DIR_LOCALE.end())
{ {
svLanguage = DIR_LOCALE[0]; svLanguage = DIR_LOCALE[0];
} }
if (std::find(DIR_CONTEXT.begin(), DIR_CONTEXT.end(), svContext) == DIR_CONTEXT.end()) if (std::find(DIR_TARGET.begin(), DIR_TARGET.end(), svTarget) == DIR_TARGET.end())
{ {
svContext = DIR_CONTEXT[0]; svTarget = DIR_TARGET[0];
} }
VPKPair_t vPair; VPKPair_t vPair;
vPair.m_svBlockName = fmt::format("{:s}_{:s}.bsp.pak000_{:03d}{:s}", svContext, svPakName, nPatch, ".vpk"); vPair.m_svBlockName = fmt::format("{:s}_{:s}.bsp.pak000_{:03d}{:s}", svTarget, svLevel, nPatch, ".vpk");
vPair.m_svDirectoryName = fmt::format("{:s}{:s}_{:s}.bsp.pak000_{:s}", svLanguage, svContext, svPakName, "dir.vpk"); vPair.m_svDirectoryName = fmt::format("{:s}{:s}_{:s}.bsp.pak000_{:s}", svLanguage, svTarget, svLevel, "dir.vpk");
return vPair; return vPair;
} }
@ -401,7 +404,7 @@ void CPackedStore::BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const st
for (const VPKEntryBlock_t& vEntry : vBlock) for (const VPKEntryBlock_t& vEntry : vBlock)
{ {
const VPKChunkDescriptor_t& vDescriptor = vEntry.m_vChunks[0]; const VPKChunkDescriptor_t& vDescriptor = vEntry.m_vFragments[0];
jEntry[vEntry.m_svEntryPath] = jEntry[vEntry.m_svEntryPath] =
{ {
{ "preloadSize", vEntry.m_iPreloadSize }, { "preloadSize", vEntry.m_iPreloadSize },
@ -454,33 +457,33 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile)
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: packs all files from specified path into VPK file // Purpose: packs all files from workspace path into VPK file
// Input : &vPair - // Input : &vPair -
// &svPathIn - // &svWorkspace -
// &svPathOut - // &svBuildPath -
// bManifestOnly - // bManifestOnly -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const string& svPathOut, bool bManifestOnly) void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly)
{ {
CIOStream writer(svPathOut + vPair.m_svBlockName, CIOStream::Mode_t::WRITE); CIOStream writer(svBuildPath + vPair.m_svBlockName, CIOStream::Mode_t::WRITE);
vector<string> vPaths; vector<string> vPaths;
vector<VPKEntryBlock_t> vEntryBlocks; vector<VPKEntryBlock_t> vEntryBlocks;
const nlohmann::json jManifest = GetManifest(svPathIn, GetSourceName(vPair.m_svDirectoryName)); const nlohmann::json jManifest = GetManifest(svWorkspace, GetLevelName(vPair.m_svDirectoryName));
GetIgnoreList(svPathIn); GetIgnoreList(svWorkspace);
if (bManifestOnly) if (bManifestOnly)
{ {
vPaths = GetEntryPaths(svPathIn, jManifest); vPaths = GetEntryPaths(svWorkspace, jManifest);
} }
else // Pack all files in workspace. else // Pack all files in workspace.
{ {
vPaths = GetEntryPaths(svPathIn); vPaths = GetEntryPaths(svWorkspace);
} }
uint64_t nSharedTotal = 0i64; uint64_t nSharedTotal = 0;
uint32_t nSharedCount = 0i32; uint32_t nSharedCount = 0;
for (size_t i = 0, ps = vPaths.size(); i < ps; i++) for (size_t i = 0, ps = vPaths.size(); i < ps; i++)
{ {
@ -488,10 +491,10 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const
CIOStream reader(svPath, CIOStream::Mode_t::READ); CIOStream reader(svPath, CIOStream::Mode_t::READ);
if (reader.IsReadable()) if (reader.IsReadable())
{ {
const string svDestPath = StringReplaceC(svPath, svPathIn, ""); const string svDestPath = StringReplaceC(svPath, svWorkspace, "");
uint16_t iPreloadSize = 0i16; uint16_t iPreloadSize = 0;
uint32_t nLoadFlags = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE); uint32_t nLoadFlags = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE);
uint16_t nTextureFlags = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT); // !TODO: Reverse these. uint16_t nTextureFlags = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
bool bUseCompression = true; bool bUseCompression = true;
bool bUseDataSharing = true; bool bUseDataSharing = true;
@ -519,12 +522,12 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const
vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), iPreloadSize, 0, nLoadFlags, nTextureFlags, svDestPath)); vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), iPreloadSize, 0, nLoadFlags, nTextureFlags, svDestPath));
VPKEntryBlock_t& vEntry = vEntryBlocks[i]; VPKEntryBlock_t& vEntry = vEntryBlocks[i];
for (size_t j = 0, es = vEntry.m_vChunks.size(); j < es; j++) for (size_t j = 0, es = vEntry.m_vFragments.size(); j < es; j++)
{ {
VPKChunkDescriptor_t& vDescriptor = vEntry.m_vChunks[j]; VPKChunkDescriptor_t& vDescriptor = vEntry.m_vFragments[j];
reader.Read(s_EntryBuf, vDescriptor.m_nUncompressedSize); reader.Read(s_EntryBuf, vDescriptor.m_nUncompressedSize);
vDescriptor.m_nArchiveOffset = writer.GetPosition(); vDescriptor.m_nPackFileOffset = writer.GetPosition();
if (bUseDataSharing) if (bUseDataSharing)
{ {
@ -533,7 +536,7 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const
if (!p.second) // Map to existing chunk to avoid having copies of the same data. if (!p.second) // Map to existing chunk to avoid having copies of the same data.
{ {
DevMsg(eDLL_T::FS, "Mapping chunk '%zu' ('%s') to existing chunk at '0x%llx'\n", j, svEntryHash.c_str(), p.first->second.m_nArchiveOffset); DevMsg(eDLL_T::FS, "Mapping chunk '%zu' ('%s') to existing chunk at '0x%llx'\n", j, svEntryHash.c_str(), p.first->second.m_nPackFileOffset);
vDescriptor = p.first->second; vDescriptor = p.first->second;
nSharedTotal += vDescriptor.m_nCompressedSize; nSharedTotal += vDescriptor.m_nCompressedSize;
@ -573,15 +576,15 @@ void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const
memset(s_EntryBuf, '\0', sizeof(s_EntryBuf)); memset(s_EntryBuf, '\0', sizeof(s_EntryBuf));
VPKDir_t vDir = VPKDir_t(); VPKDir_t vDir = VPKDir_t();
vDir.Build(svPathOut + vPair.m_svDirectoryName, vEntryBlocks); vDir.Build(svBuildPath + vPair.m_svDirectoryName, vEntryBlocks);
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: extracts all files from specified VPK file // Purpose: rebuilds manifest and extracts all files from specified VPK file
// Input : &vDir - // Input : &vDir -
// &svPathOut - // &svWorkspace -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStore::UnpackAll(const VPKDir_t& vDir, const string& svPathOut) void CPackedStore::UnpackWorkspace(const VPKDir_t& vDir, const string& svWorkspace)
{ {
if (vDir.m_vHeader.m_nHeaderMarker != VPK_HEADER_MARKER || if (vDir.m_vHeader.m_nHeaderMarker != VPK_HEADER_MARKER ||
vDir.m_vHeader.m_nMajorVersion != VPK_MAJOR_VERSION || vDir.m_vHeader.m_nMajorVersion != VPK_MAJOR_VERSION ||
@ -590,13 +593,13 @@ void CPackedStore::UnpackAll(const VPKDir_t& vDir, const string& svPathOut)
Error(eDLL_T::FS, NO_ERROR, "Unsupported VPK directory file (invalid header criteria)\n"); Error(eDLL_T::FS, NO_ERROR, "Unsupported VPK directory file (invalid header criteria)\n");
return; return;
} }
BuildManifest(vDir.m_vEntryBlocks, svPathOut, GetSourceName(vDir.m_svDirPath)); BuildManifest(vDir.m_vEntryBlocks, svWorkspace, GetLevelName(vDir.m_svDirPath));
for (size_t i = 0, fs = vDir.m_vPackFile.size(); i < fs; i++) for (size_t i = 0, fs = vDir.m_vPackFile.size(); i < fs; i++)
{ {
const fs::path fspVpkPath(vDir.m_svDirPath); const fs::path fspVpkPath(vDir.m_svDirPath);
const string svPath = fspVpkPath.parent_path().u8string() + '\\' + vDir.m_vPackFile[i]; const string svPath = fspVpkPath.parent_path().u8string() + '\\' + vDir.m_vPackFile[i];
CIOStream iStream(svPath, CIOStream::Mode_t::READ); // Create stream to read from each archive. CIOStream iStream(svPath, CIOStream::Mode_t::READ); // Create stream to read from each pack file.
for (size_t j = 0, es = vDir.m_vEntryBlocks.size(); j < es; j++) for (size_t j = 0, es = vDir.m_vEntryBlocks.size(); j < es; j++)
{ {
@ -607,7 +610,7 @@ void CPackedStore::UnpackAll(const VPKDir_t& vDir, const string& svPathOut)
} }
else // Chunk belongs to this block. else // Chunk belongs to this block.
{ {
const string svFilePath = CreateDirectories(svPathOut + vBlock.m_svEntryPath); const string svFilePath = CreateDirectories(svWorkspace + vBlock.m_svEntryPath);
CIOStream oStream(svFilePath, CIOStream::Mode_t::WRITE); CIOStream oStream(svFilePath, CIOStream::Mode_t::WRITE);
if (!oStream.IsWritable()) if (!oStream.IsWritable())
@ -617,12 +620,12 @@ void CPackedStore::UnpackAll(const VPKDir_t& vDir, const string& svPathOut)
} }
DevMsg(eDLL_T::FS, "Unpacking entry '%zu' from block '%zu' ('%s')\n", j, i, vBlock.m_svEntryPath.c_str()); DevMsg(eDLL_T::FS, "Unpacking entry '%zu' from block '%zu' ('%s')\n", j, i, vBlock.m_svEntryPath.c_str());
for (size_t k = 0, cs = vBlock.m_vChunks.size(); k < cs; k++) for (size_t k = 0, cs = vBlock.m_vFragments.size(); k < cs; k++)
{ {
const VPKChunkDescriptor_t& vChunk = vBlock.m_vChunks[k]; const VPKChunkDescriptor_t& vChunk = vBlock.m_vFragments[k];
m_nChunkCount++; m_nChunkCount++;
iStream.SetPosition(vChunk.m_nArchiveOffset); iStream.SetPosition(vChunk.m_nPackFileOffset);
iStream.Read(s_EntryBuf, vChunk.m_nCompressedSize); iStream.Read(s_EntryBuf, vChunk.m_nCompressedSize);
if (vChunk.m_bIsCompressed) if (vChunk.m_bIsCompressed)
@ -652,7 +655,7 @@ void CPackedStore::UnpackAll(const VPKDir_t& vDir, const string& svPathOut)
} }
} }
if (m_nChunkCount == vBlock.m_vChunks.size()) // Only validate after last entry in block had been written. if (m_nChunkCount == vBlock.m_vFragments.size()) // Only validate after last entry in block had been written.
{ {
m_nChunkCount = 0; m_nChunkCount = 0;
m_nCrc32_Internal = vBlock.m_nFileCRC; m_nCrc32_Internal = vBlock.m_nFileCRC;
@ -677,45 +680,45 @@ VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* pReader, string svEntryPath)
StringReplace(svEntryPath, "\\", "/"); // Flip windows-style backslash to forward slash. StringReplace(svEntryPath, "\\", "/"); // Flip windows-style backslash to forward slash.
StringReplace(svEntryPath, " /", "" ); // Remove space character representing VPK root. StringReplace(svEntryPath, " /", "" ); // Remove space character representing VPK root.
this->m_svEntryPath = svEntryPath; // Set the entry path. m_svEntryPath = svEntryPath; // Set the entry path.
pReader->Read<uint32_t>(this->m_nFileCRC); // pReader->Read<uint32_t>(m_nFileCRC); //
pReader->Read<uint16_t>(this->m_iPreloadSize); // pReader->Read<uint16_t>(m_iPreloadSize); //
pReader->Read<uint16_t>(this->m_iPackFileIndex); // pReader->Read<uint16_t>(m_iPackFileIndex); //
do // Loop through all chunks in the entry and push them to the vector. do // Loop through all chunks in the entry and add to list.
{ {
VPKChunkDescriptor_t entry(pReader); VPKChunkDescriptor_t entry(pReader);
this->m_vChunks.push_back(entry); m_vFragments.push_back(entry);
} while (pReader->Read<uint16_t>() != UINT16_MAX); } while (pReader->Read<uint16_t>() != PACKFILEINDEX_END);
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: 'VPKEntryBlock_t' memory constructor // Purpose: 'VPKEntryBlock_t' memory constructor
// Input : &vData - // Input : &vData -
// nOffset - // nOffset -
// nPreloadSize - // iPreloadSize -
// nArchiveIndex - // iPackFileIndex -
// nLoadFlags - // nLoadFlags -
// nTextureFlags - // nTextureFlags -
// &svBlockPath - // &svEntryPath -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKEntryBlock_t::VPKEntryBlock_t(const vector<uint8_t> &vData, int64_t nOffset, uint16_t nPreloadSize, VPKEntryBlock_t::VPKEntryBlock_t(const vector<uint8_t> &vData, int64_t nOffset, uint16_t iPreloadSize,
uint16_t nArchiveIndex, uint32_t nLoadFlags, uint16_t nTextureFlags, const string& svEntryPath) uint16_t iPackFileIndex, uint32_t nLoadFlags, uint16_t nTextureFlags, const string& svEntryPath)
{ {
m_nFileCRC = crc32::update(NULL, vData.data(), vData.size()); m_nFileCRC = crc32::update(NULL, vData.data(), vData.size());
m_iPreloadSize = nPreloadSize; m_iPreloadSize = iPreloadSize;
m_iPackFileIndex = nArchiveIndex; m_iPackFileIndex = iPackFileIndex;
m_svEntryPath = svEntryPath; m_svEntryPath = svEntryPath;
size_t nEntryCount = (vData.size() + ENTRY_MAX_LEN - 1) / ENTRY_MAX_LEN; size_t nFragmentCount = (vData.size() + ENTRY_MAX_LEN - 1) / ENTRY_MAX_LEN;
size_t nDataSize = vData.size(); size_t nFileSize = vData.size();
int64_t nCurrentOffset = nOffset; int64_t nCurrentOffset = nOffset;
for (size_t i = 0; i < nEntryCount; i++) // Fragment data into 1 MiB chunks. for (size_t i = 0; i < nFragmentCount; i++) // Fragment data into 1 MiB chunks.
{ {
size_t nSize = std::min<uint64_t>(ENTRY_MAX_LEN, nDataSize); size_t nSize = std::min<uint64_t>(ENTRY_MAX_LEN, nFileSize);
nDataSize -= nSize; nFileSize -= nSize;
m_vChunks.push_back(VPKChunkDescriptor_t(nLoadFlags, nTextureFlags, nCurrentOffset, nSize, nSize)); m_vFragments.push_back(VPKChunkDescriptor_t(nLoadFlags, nTextureFlags, nCurrentOffset, nSize, nSize));
nCurrentOffset += nSize; nCurrentOffset += nSize;
} }
} }
@ -726,12 +729,12 @@ VPKEntryBlock_t::VPKEntryBlock_t(const vector<uint8_t> &vData, int64_t nOffset,
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKChunkDescriptor_t::VPKChunkDescriptor_t(CIOStream* pReader) VPKChunkDescriptor_t::VPKChunkDescriptor_t(CIOStream* pReader)
{ {
pReader->Read<uint32_t>(this->m_nLoadFlags); // pReader->Read<uint32_t>(m_nLoadFlags); //
pReader->Read<uint16_t>(this->m_nTextureFlags); // pReader->Read<uint16_t>(m_nTextureFlags); //
pReader->Read<uint64_t>(this->m_nArchiveOffset); // pReader->Read<uint64_t>(m_nPackFileOffset); //
pReader->Read<uint64_t>(this->m_nCompressedSize); // pReader->Read<uint64_t>(m_nCompressedSize); //
pReader->Read<uint64_t>(this->m_nUncompressedSize); // pReader->Read<uint64_t>(m_nUncompressedSize); //
this->m_bIsCompressed = (this->m_nCompressedSize != this->m_nUncompressedSize); m_bIsCompressed = (m_nCompressedSize != m_nUncompressedSize);
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -747,7 +750,7 @@ VPKChunkDescriptor_t::VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextur
{ {
m_nLoadFlags = nLoadFlags; m_nLoadFlags = nLoadFlags;
m_nTextureFlags = nTextureFlags; m_nTextureFlags = nTextureFlags;
m_nArchiveOffset = nArchiveOffset; m_nPackFileOffset = nArchiveOffset;
m_nCompressedSize = nCompressedSize; m_nCompressedSize = nCompressedSize;
m_nUncompressedSize = nUncompressedSize; m_nUncompressedSize = nUncompressedSize;
@ -755,33 +758,33 @@ VPKChunkDescriptor_t::VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextur
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: 'VPKDir_t' file constructor // Purpose: 'VPKDir_t' file constructor
// Input : &szPath - // Input : &svPath -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKDir_t::VPKDir_t(const string& svPath) VPKDir_t::VPKDir_t(const string& svPath)
{ {
CIOStream reader(svPath, CIOStream::Mode_t::READ); CIOStream reader(svPath, CIOStream::Mode_t::READ);
reader.Read<uint32_t>(this->m_vHeader.m_nHeaderMarker); reader.Read<uint32_t>(m_vHeader.m_nHeaderMarker);
reader.Read<uint16_t>(this->m_vHeader.m_nMajorVersion); // reader.Read<uint16_t>(m_vHeader.m_nMajorVersion); //
reader.Read<uint16_t>(this->m_vHeader.m_nMinorVersion); // reader.Read<uint16_t>(m_vHeader.m_nMinorVersion); //
reader.Read<uint32_t>(this->m_vHeader.m_nDirectorySize); // reader.Read<uint32_t>(m_vHeader.m_nDirectorySize); //
reader.Read<uint32_t>(this->m_nFileDataSize); // reader.Read<uint32_t>(m_nFileDataSize); //
this->m_vEntryBlocks = g_pPackedStore->GetEntryBlocks(&reader); m_vEntryBlocks = g_pPackedStore->GetEntryBlocks(&reader);
this->m_svDirPath = svPath; // Set path to vpk directory file. m_svDirPath = svPath; // Set path to vpk directory file.
for (VPKEntryBlock_t vEntry : this->m_vEntryBlocks) for (VPKEntryBlock_t vEntry : m_vEntryBlocks)
{ {
if (vEntry.m_iPackFileIndex > this->m_iPackFileCount) if (vEntry.m_iPackFileIndex > m_nPackFileCount)
{ {
this->m_iPackFileCount = vEntry.m_iPackFileIndex; m_nPackFileCount = vEntry.m_iPackFileIndex;
} }
} }
for (uint16_t i = 0; i < this->m_iPackFileCount + 1; i++) for (uint16_t i = 0; i < m_nPackFileCount + 1; i++)
{ {
string svArchivePath = g_pPackedStore->GetPackFile(svPath, i); string svArchivePath = g_pPackedStore->GetPackFile(svPath, i);
this->m_vPackFile.push_back(svArchivePath); m_vPackFile.push_back(svArchivePath);
} }
} }
@ -796,11 +799,11 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
auto vMap = std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>(); auto vMap = std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>();
uint64_t nDescriptors = 0; uint64_t nDescriptors = 0;
writer.Write<uint32_t>(this->m_vHeader.m_nHeaderMarker); writer.Write<uint32_t>(m_vHeader.m_nHeaderMarker);
writer.Write<uint16_t>(this->m_vHeader.m_nMajorVersion); writer.Write<uint16_t>(m_vHeader.m_nMajorVersion);
writer.Write<uint16_t>(this->m_vHeader.m_nMinorVersion); writer.Write<uint16_t>(m_vHeader.m_nMinorVersion);
writer.Write<uint32_t>(this->m_vHeader.m_nDirectorySize); writer.Write<uint32_t>(m_vHeader.m_nDirectorySize);
writer.Write<uint32_t>(this->m_vHeader.m_nSignatureSize); writer.Write<uint32_t>(m_vHeader.m_nSignatureSize);
for (const VPKEntryBlock_t& vBlock : vEntryBlocks) for (const VPKEntryBlock_t& vBlock : vEntryBlocks)
{ {
@ -836,14 +839,14 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
writer.Write(vEntry.m_iPreloadSize); writer.Write(vEntry.m_iPreloadSize);
writer.Write(vEntry.m_iPackFileIndex); writer.Write(vEntry.m_iPackFileIndex);
for (size_t i = 0, nc = vEntry.m_vChunks.size(); i < nc; i++) for (size_t i = 0, nc = vEntry.m_vFragments.size(); i < nc; i++)
{ {
/*Write chunk descriptor*/ /*Write chunk descriptor*/
const VPKChunkDescriptor_t* pDescriptor = &vEntry.m_vChunks[i]; const VPKChunkDescriptor_t* pDescriptor = &vEntry.m_vFragments[i];
writer.Write(pDescriptor->m_nLoadFlags); writer.Write(pDescriptor->m_nLoadFlags);
writer.Write(pDescriptor->m_nTextureFlags); writer.Write(pDescriptor->m_nTextureFlags);
writer.Write(pDescriptor->m_nArchiveOffset); writer.Write(pDescriptor->m_nPackFileOffset);
writer.Write(pDescriptor->m_nCompressedSize); writer.Write(pDescriptor->m_nCompressedSize);
writer.Write(pDescriptor->m_nUncompressedSize); writer.Write(pDescriptor->m_nUncompressedSize);
@ -852,9 +855,9 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
const ushort s = 0; const ushort s = 0;
writer.Write(s); writer.Write(s);
} }
else else // Mark end of entry.
{ {
const ushort s = UINT16_MAX; const ushort s = PACKFILEINDEX_END;
writer.Write(s); writer.Write(s);
} }
nDescriptors++; nDescriptors++;
@ -868,11 +871,14 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
m_vHeader.m_nDirectorySize = static_cast<uint32_t>(writer.GetPosition() - sizeof(VPKDirHeader_t)); m_vHeader.m_nDirectorySize = static_cast<uint32_t>(writer.GetPosition() - sizeof(VPKDirHeader_t));
writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nDirectorySize)); writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nDirectorySize));
writer.Write(this->m_vHeader.m_nDirectorySize); writer.Write(m_vHeader.m_nDirectorySize);
writer.Write(0); writer.Write(0);
DevMsg(eDLL_T::FS, "*** Build directory totaling '%zu' bytes with '%zu' entries and '%zu' descriptors\n", DevMsg(eDLL_T::FS, "*** Build directory totaling '%zu' bytes with '%zu' entries and '%zu' descriptors\n",
size_t(sizeof(VPKDirHeader_t) + m_vHeader.m_nDirectorySize), vEntryBlocks.size(), nDescriptors); size_t(sizeof(VPKDirHeader_t) + m_vHeader.m_nDirectorySize), vEntryBlocks.size(), nDescriptors);
} }
///////////////////////////////////////////////////////////////////////////////
//-----------------------------------------------------------------------------
// Singleton
//-----------------------------------------------------------------------------
CPackedStore* g_pPackedStore = new CPackedStore(); CPackedStore* g_pPackedStore = new CPackedStore();

53
r5dev/vpklib/packedstore.h
View File

@ -16,11 +16,12 @@ constexpr unsigned int VPK_MAJOR_VERSION = 2;
constexpr unsigned int VPK_MINOR_VERSION = 3; constexpr unsigned int VPK_MINOR_VERSION = 3;
constexpr unsigned int VPK_DICT_SIZE = 20; constexpr unsigned int VPK_DICT_SIZE = 20;
constexpr int ENTRY_MAX_LEN = 1024 * 1024; constexpr int ENTRY_MAX_LEN = 1024 * 1024;
constexpr int PACKFILEINDEX_END = 0xffff;
static const std::regex BLOCK_REGEX{ R"(pak000_([0-9]{3}))" }; static const std::regex BLOCK_REGEX{ R"(pak000_([0-9]{3}))" };
static const std::regex DIR_REGEX{ R"((?:.*\/)?([^_]*_)(.*)(.bsp.pak000_dir).*)" }; static const std::regex DIR_REGEX{ R"((?:.*\/)?([^_]*_)(.*)(.bsp.pak000_dir).*)" };
static const vector<string> DIR_CONTEXT = static const vector<string> DIR_TARGET =
{ {
"server", "server",
"client" "client"
@ -43,7 +44,7 @@ static const vector<string> DIR_LOCALE =
enum class EPackedLoadFlags : int enum class EPackedLoadFlags : int
{ {
LOAD_NONE, LOAD_NONE,
LOAD_VISIBLE = 1 << 0, // FileSystem visibility? LOAD_VISIBLE = 1 << 0, // Visible to FileSystem.
LOAD_CACHE = 1 << 8, // Only set for assets not stored in the depot directory. LOAD_CACHE = 1 << 8, // Only set for assets not stored in the depot directory.
LOAD_TEXTURE_UNK0 = 1 << 18, LOAD_TEXTURE_UNK0 = 1 << 18,
LOAD_TEXTURE_UNK1 = 1 << 19, LOAD_TEXTURE_UNK1 = 1 << 19,
@ -94,14 +95,14 @@ struct VPKChunkDescriptor_t
{ {
uint32_t m_nLoadFlags; // Load flags. uint32_t m_nLoadFlags; // Load flags.
uint16_t m_nTextureFlags; // Texture flags (only used if the entry is a vtf). uint16_t m_nTextureFlags; // Texture flags (only used if the entry is a vtf).
uint64_t m_nArchiveOffset; // Offset in archive. uint64_t m_nPackFileOffset; // Offset in pack file.
uint64_t m_nCompressedSize; // Compressed size of chunk. uint64_t m_nCompressedSize; // Compressed size of chunk.
uint64_t m_nUncompressedSize; // Uncompressed size of chunk. uint64_t m_nUncompressedSize; // Uncompressed size of chunk.
bool m_bIsCompressed = false; bool m_bIsCompressed = false;
VPKChunkDescriptor_t(){}; VPKChunkDescriptor_t(){};
VPKChunkDescriptor_t(CIOStream* pReader); VPKChunkDescriptor_t(CIOStream* pReader);
VPKChunkDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nArchiveOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize); VPKChunkDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize);
}; };
struct VPKEntryBlock_t struct VPKEntryBlock_t
@ -109,11 +110,11 @@ struct VPKEntryBlock_t
uint32_t m_nFileCRC; // Crc32 for the uncompressed entry. uint32_t m_nFileCRC; // Crc32 for the uncompressed entry.
uint16_t m_iPreloadSize; // Preload bytes. uint16_t m_iPreloadSize; // Preload bytes.
uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry. uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry.
vector<VPKChunkDescriptor_t> m_vChunks; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks). vector<VPKChunkDescriptor_t> m_vFragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks).
string m_svEntryPath; // Path to entry within vpk. string m_svEntryPath; // Path to entry within vpk.
VPKEntryBlock_t(CIOStream* pReader, string svEntryPath); VPKEntryBlock_t(CIOStream* pReader, string svEntryPath);
VPKEntryBlock_t(const vector<uint8_t>& vData, int64_t nOffset, uint16_t nPreloadData, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, const string& svEntryPath); VPKEntryBlock_t(const vector<uint8_t>& vData, int64_t nOffset, uint16_t iPreloadSize, uint16_t iPackFileIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, const string& svEntryPath);
}; };
struct VPKDirHeader_t struct VPKDirHeader_t
@ -130,8 +131,8 @@ struct VPKDir_t
VPKDirHeader_t m_vHeader; // Dir header. VPKDirHeader_t m_vHeader; // Dir header.
uint32_t m_nFileDataSize; // File data section size. uint32_t m_nFileDataSize; // File data section size.
vector<VPKEntryBlock_t> m_vEntryBlocks; // Vector of entry blocks. vector<VPKEntryBlock_t> m_vEntryBlocks; // Vector of entry blocks.
uint16_t m_iPackFileCount; // Highest archive index (archive count-1). uint16_t m_nPackFileCount; // Number of pack patches (pack file count-1).
vector<string> m_vPackFile; // Vector of archive file names. vector<string> m_vPackFile; // Vector of pack file names.
string m_svDirPath; // Path to vpk_dir file. string m_svDirPath; // Path to vpk_dir file.
VPKDir_t(const string& svPath); VPKDir_t(const string& svPath);
@ -142,7 +143,7 @@ struct VPKDir_t
struct VPKPair_t struct VPKPair_t
{ {
string m_svBlockName; // !TODO: Multi-pak support. string m_svBlockName;
string m_svDirectoryName; string m_svDirectoryName;
}; };
@ -153,7 +154,7 @@ public:
void InitLzDecompParams(void); void InitLzDecompParams(void);
VPKDir_t GetDirectoryFile(const string& svDirectoryFile, bool bSanitizeName) const; VPKDir_t GetDirectoryFile(const string& svDirectoryFile, bool bSanitizeName) const;
string GetPackFile(const string& svPackDirFile, uint16_t iArchiveIndex) const; string GetPackFile(const string& svPackDirFile, uint16_t iPackFileIndex) const;
lzham_compress_level GetCompressionLevel(void) const; lzham_compress_level GetCompressionLevel(void) const;
vector<VPKEntryBlock_t> GetEntryBlocks(CIOStream* pReader) const; vector<VPKEntryBlock_t> GetEntryBlocks(CIOStream* pReader) const;
@ -161,34 +162,34 @@ public:
vector<string> GetEntryPaths(const string& svPathIn, const nlohmann::json& jManifest) const; vector<string> GetEntryPaths(const string& svPathIn, const nlohmann::json& jManifest) const;
string GetNameParts(const string& svDirectoryName, int nCaptureGroup) const; string GetNameParts(const string& svDirectoryName, int nCaptureGroup) const;
string GetSourceName(const string& svDirectoryName) const; string GetLevelName(const string& svDirectoryName) const;
nlohmann::json GetManifest(const string& svWorkSpace, const string& svManifestName) const; nlohmann::json GetManifest(const string& svWorkspace, const string& svManifestName) const;
vector<string> GetIgnoreList(const string& svWorkSpace) const; vector<string> GetIgnoreList(const string& svWorkspace) const;
string FormatEntryPath(string svName, const string& svPath, const string& svExtension) const; string FormatEntryPath(string svName, const string& svPath, const string& svExtension) const;
string StripLocalePrefix(const string& svDirectoryFile) const; string StripLocalePrefix(const string& svDirectoryFile) const;
VPKPair_t BuildFileName(string svLanguage, string svContext, const string& svPakName, int nPatch) const; VPKPair_t BuildFileName(string svLanguage, string svTarget, const string& svLevel, int nPatch) const;
void BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const string& svWorkSpace, const string& svManifestName) const; void BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const string& svWorkSpace, const string& svManifestName) const;
void PackAll(const VPKPair_t& vPair, const string& svPathIn, const string& svPathOut, bool bManifestOnly); void PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly);
void UnpackAll(const VPKDir_t& vDir, const string& svPathOut = ""); void UnpackWorkspace(const VPKDir_t& vDir, const string& svPathOut = "");
void ValidateAdler32PostDecomp(const string& svDirAsset); void ValidateAdler32PostDecomp(const string& svDirAsset);
void ValidateCRC32PostDecomp(const string& svDirAsset); void ValidateCRC32PostDecomp(const string& svDirAsset);
private: private:
size_t m_nChunkCount {}; // Entry per-block incrementor. size_t m_nChunkCount; // The number of patches (multi-pack file).
lzham_uint32 m_nAdler32_Internal{}; // Internal operation Adler32 file checksum. lzham_uint32 m_nAdler32_Internal; // Internal operation Adler32 file checksum.
lzham_uint32 m_nAdler32 {}; // Pre/post operation Adler32 file checksum. lzham_uint32 m_nAdler32; // Pre/post operation Adler32 file checksum.
lzham_uint32 m_nCrc32_Internal {}; // Internal operation Crc32 file checksum. lzham_uint32 m_nCrc32_Internal; // Internal operation Crc32 file checksum.
lzham_uint32 m_nCrc32 {}; // Pre/post operation Crc32 file checksum. lzham_uint32 m_nCrc32; // Pre/post operation Crc32 file checksum.
lzham_compress_params m_lzCompParams {}; // LZham decompression parameters. lzham_compress_params m_lzCompParams; // LZham decompression parameters.
lzham_compress_status_t m_lzCompStatus {}; // LZham compression status. lzham_compress_status_t m_lzCompStatus; // LZham compression status.
lzham_decompress_params m_lzDecompParams {}; // LZham decompression parameters. lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
lzham_decompress_status_t m_lzDecompStatus {}; // LZham decompression status. lzham_decompress_status_t m_lzDecompStatus; // LZham decompression status.
std::unordered_map<string, VPKChunkDescriptor_t&> m_mChunkHashMap{}; std::unordered_map<string, VPKChunkDescriptor_t&> m_mChunkHashMap;
}; };
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
extern CPackedStore* g_pPackedStore; extern CPackedStore* g_pPackedStore;

12
r5dev/vstdlib/callback.cpp
View File

@ -521,8 +521,8 @@ void RTech_Decompress_f(const CCommand& args)
===================== =====================
VPK_Pack_f VPK_Pack_f
Compresses new VPK files and Packs VPK files into
dumps the output to '\vpk'. 'PLATFORM' VPK directory.
===================== =====================
*/ */
void VPK_Pack_f(const CCommand& args) void VPK_Pack_f(const CCommand& args)
@ -538,7 +538,7 @@ void VPK_Pack_f(const CCommand& args)
DevMsg(eDLL_T::FS, "*** Starting VPK build command for: '%s'\n", vPair.m_svDirectoryName.c_str()); DevMsg(eDLL_T::FS, "*** Starting VPK build command for: '%s'\n", vPair.m_svDirectoryName.c_str());
std::thread th([&] { g_pPackedStore->PackAll(vPair, fs_packedstore_workspace->GetString(), "vpk/", (args.ArgC() > 4)); }); std::thread th([&] { g_pPackedStore->PackWorkspace(vPair, fs_packedstore_workspace->GetString(), "vpk/", (args.ArgC() > 4)); });
th.join(); th.join();
std::chrono::milliseconds msEnd = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()); std::chrono::milliseconds msEnd = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch());
@ -552,8 +552,8 @@ void VPK_Pack_f(const CCommand& args)
===================== =====================
VPK_Unpack_f VPK_Unpack_f
Decompresses input VPK files and Unpacks VPK files into
dumps the output to '<mod>\vpk'. workspace directory.
===================== =====================
*/ */
void VPK_Unpack_f(const CCommand& args) void VPK_Unpack_f(const CCommand& args)
@ -570,7 +570,7 @@ void VPK_Unpack_f(const CCommand& args)
VPKDir_t vpk = g_pPackedStore->GetDirectoryFile(pArg, (args.ArgC() > 2)); VPKDir_t vpk = g_pPackedStore->GetDirectoryFile(pArg, (args.ArgC() > 2));
g_pPackedStore->InitLzDecompParams(); g_pPackedStore->InitLzDecompParams();
std::thread th([&] { g_pPackedStore->UnpackAll(vpk, ConvertToWinPath(fs_packedstore_workspace->GetString())); }); std::thread th([&] { g_pPackedStore->UnpackWorkspace(vpk, ConvertToWinPath(fs_packedstore_workspace->GetString())); });
th.join(); th.join();
std::chrono::milliseconds msEnd = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()); std::chrono::milliseconds msEnd = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch());