R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

CPackedStore refactor

Browse Source 
The CPackedStore class now heavily relies on the engine's FileSystem and KeyValues API. Since its part of the engine, it would make more sense to utilize the engine's features instead. It also allows for easier modifications.
* The logic behind the build process hasn't changed.
This commit is contained in:
Kawe Mazidjatari 2022-11-23 12:18:33 +01:00
parent 69cd41881e
commit 0ab31606eb
6 changed files with 362 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
r5dev/filesystem/basefilesystem.cpp
View File

@ -164,6 +164,28 @@ const char* CBaseFileSystem::VUnmountVPKFile(CBaseFileSystem* pFileSystem, const
return pRet; return pRet;
} }
//---------------------------------------------------------------------------------
// Purpose: reads a string until its null terminator
// Input : *pFile -
// Output : string
//---------------------------------------------------------------------------------
string CBaseFileSystem::ReadString(FileHandle_t pFile)
{
string svString;
char c = '\0';
do
{
Read(&c, sizeof(char), pFile);
if (c)
svString += c;
} while (c);
return svString;
}
void CBaseFileSystem_Attach() void CBaseFileSystem_Attach()
{ {
DetourAttach((LPVOID*)&v_CBaseFileSystem_Warning, &CBaseFileSystem::Warning); DetourAttach((LPVOID*)&v_CBaseFileSystem_Warning, &CBaseFileSystem::Warning);

2
r5dev/filesystem/basefilesystem.h
View File

@ -14,6 +14,8 @@ public:
static VPKData_t* VMountVPKFile(CBaseFileSystem* pFileSystem, const char* pszVpkPath); static VPKData_t* VMountVPKFile(CBaseFileSystem* pFileSystem, const char* pszVpkPath);
static const char* VUnmountVPKFile(CBaseFileSystem* pFileSystem, const char* pszVpkPath); static const char* VUnmountVPKFile(CBaseFileSystem* pFileSystem, const char* pszVpkPath);
string ReadString(FileHandle_t pFile);
protected: protected:
//---------------------------------------------------------------------------- //----------------------------------------------------------------------------
// Purpose: Functions implementing basic file system behavior. // Purpose: Functions implementing basic file system behavior.

1
r5dev/public/ifilesystem.h
View File

@ -364,6 +364,7 @@ public:
TYPE_SOUNDEMITTER, TYPE_SOUNDEMITTER,
TYPE_SOUNDSCAPE, TYPE_SOUNDSCAPE,
TYPE_SOUNDOPERATORS, TYPE_SOUNDOPERATORS,
TYPE_COMMON,
NUM_PRELOAD_TYPES NUM_PRELOAD_TYPES
}; };

543
r5dev/vpklib/packedstore.cpp
View File

@ -18,6 +18,8 @@
#include "mathlib/adler32.h" #include "mathlib/adler32.h"
#include "mathlib/crc32.h" #include "mathlib/crc32.h"
#include "mathlib/sha1.h" #include "mathlib/sha1.h"
#include "filesystem/filesystem.h"
#include "vpc/keyvalues.h"
#include "vpklib/packedstore.h" #include "vpklib/packedstore.h"
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -112,10 +114,7 @@ VPKDir_t CPackedStore::GetDirectoryFile(const string& svPackDirFile, bool bSanit
string CPackedStore::GetPackFile(const string& svPackDirFile, uint16_t iPackFileIndex) const string CPackedStore::GetPackFile(const string& svPackDirFile, uint16_t iPackFileIndex) const
{ {
string svPackChunkFile = StripLocalePrefix(svPackDirFile); string svPackChunkFile = StripLocalePrefix(svPackDirFile);
ostringstream oss; string svPackChunkIndex = fmt::format("{:s}{:03d}", "pak000_", iPackFileIndex);
oss << std::setw(3) << std::setfill('0') << iPackFileIndex;
string svPackChunkIndex = "pak000_" + oss.str();
StringReplace(svPackChunkFile, "pak000_dir", svPackChunkIndex); StringReplace(svPackChunkFile, "pak000_dir", svPackChunkIndex);
return svPackChunkFile; return svPackChunkFile;
@ -145,21 +144,21 @@ lzham_compress_level CPackedStore::GetCompressionLevel(void) const
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: obtains and returns the entry block to the vector // Purpose: obtains and returns the entry block to the vector
// Input : *pReader - // Input : hDirectory -
// output : vector<VPKEntryBlock_t> // output : vector<VPKEntryBlock_t>
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
vector<VPKEntryBlock_t> CPackedStore::GetEntryBlocks(CIOStream* pReader) const vector<VPKEntryBlock_t> CPackedStore::GetEntryBlocks(FileHandle_t hDirectory) const
{ {
string svName, svPath, svExtension; string svName, svPath, svExtension;
vector<VPKEntryBlock_t> vBlocks; vector<VPKEntryBlock_t> vBlocks;
while (!(svExtension = pReader->ReadString()).empty()) while (!(svExtension = FileSystem()->ReadString(hDirectory)).empty())
{ {
while (!(svPath = pReader->ReadString()).empty()) while (!(svPath = FileSystem()->ReadString(hDirectory)).empty())
{ {
while (!(svName = pReader->ReadString()).empty()) while (!(svName = FileSystem()->ReadString(hDirectory)).empty())
{ {
const string svFilePath = FormatEntryPath(svPath, svName, svExtension); const string svFilePath = FormatEntryPath(svPath, svName, svExtension);
vBlocks.push_back(VPKEntryBlock_t(pReader, svFilePath)); vBlocks.push_back(VPKEntryBlock_t(hDirectory, svFilePath));
} }
} }
} }
@ -169,11 +168,11 @@ vector<VPKEntryBlock_t> CPackedStore::GetEntryBlocks(CIOStream* pReader) const
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: scans the input directory and returns the paths to the vector // Purpose: scans the input directory and returns the paths to the vector
// Input : &svPathIn - // Input : &svPathIn -
// Output : a string vector of all included entry paths // Output : a vpk keyvalues vector of all existing entry paths
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
vector<string> CPackedStore::GetEntryPaths(const string& svPathIn) const vector<VPKKeyValues_t> CPackedStore::GetEntryPaths(const string& svPathIn) const
{ {
vector<string> vPaths; vector<VPKKeyValues_t> vKeys;
vector<string> vIgnore = GetIgnoreList(svPathIn); vector<string> vIgnore = GetIgnoreList(svPathIn);
fs::recursive_directory_iterator dir(svPathIn), end; fs::recursive_directory_iterator dir(svPathIn), end;
@ -190,26 +189,33 @@ vector<string> CPackedStore::GetEntryPaths(const string& svPathIn) const
const string svPath = dir->path().u8string(); const string svPath = dir->path().u8string();
if (!GetExtension(svPath).empty()) if (!GetExtension(svPath).empty())
{ {
vPaths.push_back(ConvertToUnixPath(svPath)); vKeys.push_back(VPKKeyValues_t(ConvertToUnixPath(svPath)));
} }
} }
dir++; dir++;
} }
return vPaths; return vKeys;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: scans the input directory and returns the paths to the vector if path exists in manifest // Purpose: scans the input directory and returns the paths to the vector if path exists in manifest
// Input : &svPathIn - // Input : &svPathIn -
// &jManifest - // *pManifestKV -
// Output : a string vector of all included and existing entry paths // Output : a vpk keyvalues vector of all existing and included entry paths
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
vector<string> CPackedStore::GetEntryPaths(const string& svPathIn, const nlohmann::json& jManifest) const vector<VPKKeyValues_t> CPackedStore::GetEntryPaths(const string& svPathIn, KeyValues* pManifestKV) const
{ {
vector<string> vPaths; vector<VPKKeyValues_t> vKeys;
vector<string> vIgnore = GetIgnoreList(svPathIn);
if (!pManifestKV)
{
Warning(eDLL_T::FS, "Invalid VPK manifest KV; unable to build entry list\n");
return vKeys;
}
vector<string> vIgnore = GetIgnoreList(svPathIn);
fs::recursive_directory_iterator dir(svPathIn), end; fs::recursive_directory_iterator dir(svPathIn), end;
while (dir != end) while (dir != end)
{ {
const vector<string>::iterator it = std::find(vIgnore.begin(), vIgnore.end(), const vector<string>::iterator it = std::find(vIgnore.begin(), vIgnore.end(),
@ -220,29 +226,29 @@ vector<string> CPackedStore::GetEntryPaths(const string& svPathIn, const nlohman
} }
else if (dir->file_size() > 0) // Empty files are not supported. else if (dir->file_size() > 0) // Empty files are not supported.
{ {
const string svPath = dir->path().u8string(); const string svFullPath = ConvertToWinPath(dir->path().u8string());
if (!GetExtension(svPath).empty()) if (!GetExtension(svFullPath).empty())
{ {
if (!jManifest.is_null()) // Remove workspace path by offsetting it by its size.
const char* pszEntry = (svFullPath.c_str() + svPathIn.length());
KeyValues* pEntryKV = pManifestKV->FindKey(pszEntry);
if (pEntryKV)
{ {
try vKeys.push_back(VPKKeyValues_t(
{ ConvertToUnixPath(svFullPath),
const string svEntryPath = ConvertToUnixPath(svPath); pEntryKV->GetInt("preloadSize", NULL),
if (jManifest.contains(StringReplaceC(svEntryPath, svPathIn, ""))) pEntryKV->GetInt("loadFlags", static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE)),
{ pEntryKV->GetInt("textureFlags", static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT)),
vPaths.push_back(svEntryPath); pEntryKV->GetBool("useCompression", true),
} pEntryKV->GetBool("useDataSharing", true))
} );
catch (const std::exception& ex)
{
Warning(eDLL_T::FS, "Exception while reading VPK control file: '%s'\n", ex.what());
}
} }
} }
} }
dir++; dir++;
} }
return vPaths; return vKeys;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -273,34 +279,22 @@ string CPackedStore::GetLevelName(const string& svDirectoryName) const
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: gets the manifest file associated with the VPK name // Purpose: gets the manifest file associated with the VPK name (must be freed after wards)
// Input : &svWorkspace - // Input : &svWorkspace -
// &svManifestName - // &svManifestName -
// Output : parsed manifest as json // Output : KeyValues build manifest pointer
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
nlohmann::json CPackedStore::GetManifest(const string& svWorkspace, const string& svManifestName) const KeyValues* CPackedStore::GetManifest(const string& svWorkspace, const string& svManifestName) const
{ {
ostringstream ostream; string svPathOut = fmt::format("{:s}{:s}{:s}.{:s}", svWorkspace, "manifest/", svManifestName, "txt");
ostream << svWorkspace << "manifest/" << svManifestName << ".json"; KeyValues* pManifestKV = FileSystem()->LoadKeyValues(IFileSystem::TYPE_COMMON, svPathOut.c_str(), "GAME");
fs::path fsPath = fs::current_path() /= ostream.str();
nlohmann::json jsOut;
if (fs::exists(fsPath)) if (!pManifestKV)
{ {
try Warning(eDLL_T::FS, "Failed to parse VPK build manifest: '%s'\n", svPathOut.c_str());
{
ifstream iManifest(fsPath.u8string(), std::ios::binary);
jsOut = nlohmann::json::parse(iManifest);
return jsOut;
}
catch (const std::exception& ex)
{
Warning(eDLL_T::FS, "Exception while parsing VPK control file: '%s'\n", ex.what());
return jsOut;
}
} }
return jsOut;
return pManifestKV;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -310,43 +304,54 @@ nlohmann::json CPackedStore::GetManifest(const string& svWorkspace, const string
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
vector<string> CPackedStore::GetIgnoreList(const string& svWorkspace) const vector<string> CPackedStore::GetIgnoreList(const string& svWorkspace) const
{ {
fs::path fsIgnore = svWorkspace + ".vpkignore"; string svIgnore = svWorkspace + ".vpkignore";
ifstream iStream(fsIgnore); FileHandle_t hIgnoreFile = FileSystem()->Open(svIgnore.c_str(), "rb", "GAME");
if (!hIgnoreFile)
{
Warning(eDLL_T::FS, "No ignore file provided; continuing build without...\n");
return vector<string>();
}
vector<string> vIgnore; vector<string> vIgnore;
if (iStream) char szIgnore[MAX_PATH];
while (FileSystem()->ReadLine(szIgnore, sizeof(szIgnore) - 1, hIgnoreFile))
{ {
string svIgnore; if (!strstr(szIgnore, "//"))
while (std::getline(iStream, svIgnore))
{ {
string::size_type nPos = svIgnore.find("//"); if (char* pEOL = strchr(szIgnore, '\n'))
if (nPos == string::npos)
{ {
if (!svIgnore.empty() && // Null newline character.
std::find(vIgnore.begin(), vIgnore.end(), svIgnore) == vIgnore.end()) *pEOL = '\0';
if (pEOL - szIgnore > 0)
{ {
vIgnore.push_back(svIgnore); // Null carriage return.
if (*(pEOL - 1) == '\r')
{
*(pEOL - 1) = '\0';
}
} }
} }
vIgnore.push_back(szIgnore);
} }
} }
FileSystem()->Close(hIgnoreFile);
return vIgnore; return vIgnore;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: formats the file entry path // Purpose: formats the file entry path
// Input : svPath - // Input : &svPath -
// &svName - // &svName -
// &svExtension - // &svExtension -
// Output : formatted entry path // Output : formatted entry path
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
string CPackedStore::FormatEntryPath(string svPath, const string& svName, const string& svExtension) const string CPackedStore::FormatEntryPath(const string& svPath, const string& svName, const string& svExtension) const
{ {
if (!svPath.empty()) return fmt::format("{:s}{:s}{:s}.{:s}", svPath, svPath.empty() ? "" : "/", svName, svExtension);
{
svPath += '/';
}
return svPath + svName + '.' + svExtension;
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
@ -402,38 +407,52 @@ VPKPair_t CPackedStore::BuildFileName(string svLanguage, string svTarget, const
// &svWorkSpace - // &svWorkSpace -
// &svManifestName - // &svManifestName -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStore::BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const string& svWorkSpace, const string& svManifestName) const void CPackedStore::BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const string& svWorkspace, const string& svManifestName) const
{ {
nlohmann::json jEntry; KeyValues kv("BuildManifest");
KeyValues* pManifestKV = kv.FindKey("BuildManifest", true);
for (const VPKEntryBlock_t& vEntry : vBlock) for (const VPKEntryBlock_t& vEntry : vBlock)
{ {
const VPKChunkDescriptor_t& vDescriptor = vEntry.m_vFragments[0]; const VPKChunkDescriptor_t& vDescriptor = vEntry.m_vFragments[0];
jEntry[vEntry.m_svEntryPath] = KeyValues* pEntryKV = pManifestKV->FindKey(ConvertToWinPath(vEntry.m_svEntryPath).c_str(), true);
{
{ "preloadSize", vEntry.m_iPreloadSize }, pEntryKV->SetInt("preloadSize", vEntry.m_iPreloadSize);
{ "loadFlags", vDescriptor.m_nLoadFlags }, pEntryKV->SetInt("loadFlags", vDescriptor.m_nLoadFlags);
{ "textureFlags", vDescriptor.m_nTextureFlags }, pEntryKV->SetInt("textureFlags", vDescriptor.m_nTextureFlags);
{ "useCompression", vDescriptor.m_nCompressedSize != vDescriptor.m_nUncompressedSize }, pEntryKV->SetBool("useCompression", vDescriptor.m_nCompressedSize != vDescriptor.m_nUncompressedSize);
{ "useDataSharing", true } pEntryKV->SetBool("useDataSharing", true);
};
} }
string svPathOut = svWorkSpace + "manifest/"; string svPathOut = fmt::format("{:s}{:s}{:s}.{:s}", svWorkspace, "manifest/", svManifestName, "txt");
fs::create_directories(svPathOut); CUtlBuffer uBuf(0i64, 0, CUtlBuffer::TEXT_BUFFER);
ofstream oManifest(svPathOut + svManifestName + ".json"); kv.RecursiveSaveToFile(uBuf, 0);
oManifest << jEntry.dump(4);
FileSystem()->CreateDirHierarchy(string(svWorkspace + "manifest/").c_str(), "GAME");
FileSystem()->WriteFile(svPathOut.c_str(), "GAME", uBuf);
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: validates extraction result with precomputed ADLER32 hash // Purpose: validates extraction result with precomputed ADLER32 hash
// Input : &svAssetFile - // Input : &svAssetFile -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile) void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetPath)
{ {
CIOStream reader(svAssetFile, CIOStream::Mode_t::READ); FileHandle_t hAsset = FileSystem()->Open(svAssetPath.c_str(), "rb", "GAME");
m_nAdler32 = adler32::update(NULL, reader.GetData(), reader.GetSize()); if (!hAsset)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n", __FUNCTION__, svAssetPath.c_str());
return;
}
uint32_t nLen = FileSystem()->Size(hAsset);
uint8_t* pBuf = MemAllocSingleton()->Alloc<uint8_t>(nLen);
FileSystem()->Read(pBuf, nLen, hAsset);
FileSystem()->Close(hAsset);
m_nAdler32 = adler32::update(NULL, pBuf, nLen);
MemAllocSingleton()->Free(pBuf);
if (m_nAdler32 != m_nAdler32_Internal) if (m_nAdler32 != m_nAdler32_Internal)
{ {
@ -447,10 +466,22 @@ void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile)
// Purpose: validates extraction result with precomputed CRC32 hash // Purpose: validates extraction result with precomputed CRC32 hash
// Input : &svAssetFile - // Input : &svAssetFile -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile) void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetPath)
{ {
CIOStream reader(svAssetFile, CIOStream::Mode_t::READ); FileHandle_t hAsset = FileSystem()->Open(svAssetPath.c_str(), "rb", "GAME");
m_nCrc32 = crc32::update(NULL, reader.GetData(), reader.GetSize()); if (!hAsset)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n", __FUNCTION__, svAssetPath.c_str());
return;
}
uint32_t nLen = FileSystem()->Size(hAsset);
uint8_t* pBuf = MemAllocSingleton()->Alloc<uint8_t>(nLen);
FileSystem()->Read(pBuf, nLen, hAsset);
FileSystem()->Close(hAsset);
m_nCrc32 = crc32::update(NULL, pBuf, nLen);
MemAllocSingleton()->Free(pBuf);
if (m_nCrc32 != m_nCrc32_Internal) if (m_nCrc32 != m_nCrc32_Internal)
{ {
@ -469,17 +500,28 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile)
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly) void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly)
{ {
CIOStream writer(svBuildPath + vPair.m_svBlockName, CIOStream::Mode_t::WRITE); const string svPackFilePath = string(svBuildPath + vPair.m_svBlockName).c_str();
FileHandle_t hPackFile = FileSystem()->Open(svPackFilePath.c_str(), "wb", "GAME");
vector<string> vPaths; if (!hPackFile)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to write to '%s' (read-only?)\n", __FUNCTION__, svPackFilePath.c_str());
return;
}
vector<VPKKeyValues_t> vPaths;
vector<VPKEntryBlock_t> vEntryBlocks; vector<VPKEntryBlock_t> vEntryBlocks;
const nlohmann::json jManifest = GetManifest(svWorkspace, GetLevelName(vPair.m_svDirectoryName)); KeyValues* pManifestKV = nullptr;
GetIgnoreList(svWorkspace);
if (bManifestOnly) if (bManifestOnly)
{ {
vPaths = GetEntryPaths(svWorkspace, jManifest); pManifestKV = GetManifest(svWorkspace, GetLevelName(vPair.m_svDirectoryName));
vPaths = GetEntryPaths(svWorkspace, pManifestKV);
if (pManifestKV)
{
pManifestKV->DeleteThis();
}
} }
else // Pack all files in workspace. else // Pack all files in workspace.
{ {
@ -491,89 +533,77 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
for (size_t i = 0, ps = vPaths.size(); i < ps; i++) for (size_t i = 0, ps = vPaths.size(); i < ps; i++)
{ {
const string& svPath = vPaths[i]; const VPKKeyValues_t& vKeys = vPaths[i];
CIOStream reader(svPath, CIOStream::Mode_t::READ); FileHandle_t hAsset = FileSystem()->Open(vKeys.m_svEntryPath.c_str(), "rb", "GAME");
if (reader.IsReadable()) if (!hAsset)
{ {
const string svDestPath = StringReplaceC(svPath, svWorkspace, ""); Error(eDLL_T::FS, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n", __FUNCTION__, vKeys.m_svEntryPath.c_str());
uint16_t iPreloadSize = NULL; continue;
uint32_t nLoadFlags = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE); }
uint16_t nTextureFlags = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
bool bUseCompression = true;
bool bUseDataSharing = true;
if (!jManifest.is_null()) const char* szDestPath = (vKeys.m_svEntryPath.c_str() + svWorkspace.length());
uint32_t nLen = FileSystem()->Size(hAsset);
uint8_t* pBuf = MemAllocSingleton()->Alloc<uint8_t>(nLen);
FileSystem()->Read(pBuf, nLen, hAsset);
FileSystem()->Seek(hAsset, 0, FileSystemSeek_t::FILESYSTEM_SEEK_HEAD);
DevMsg(eDLL_T::FS, "Packing entry '%zu' ('%s')\n", i, szDestPath);
vEntryBlocks.push_back(VPKEntryBlock_t(pBuf, nLen, FileSystem()->Tell(hPackFile), vKeys.m_iPreloadSize, 0, vKeys.m_nLoadFlags, vKeys.m_nTextureFlags, szDestPath));
VPKEntryBlock_t& vEntry = vEntryBlocks[i];
for (size_t j = 0, es = vEntry.m_vFragments.size(); j < es; j++)
{
VPKChunkDescriptor_t& vDescriptor = vEntry.m_vFragments[j];
FileSystem()->Read(s_EntryBuf, vDescriptor.m_nCompressedSize, hAsset);
vDescriptor.m_nPackFileOffset = FileSystem()->Tell(hPackFile);
if (vKeys.m_bUseDataSharing)
{ {
try string svEntryHash = sha1(string(reinterpret_cast<char*>(s_EntryBuf), vDescriptor.m_nUncompressedSize));
auto p = m_mChunkHashMap.insert({ svEntryHash, vDescriptor });
if (!p.second) // Map to existing chunk to avoid having copies of the same data.
{ {
nlohmann::json jEntry = jManifest[svDestPath]; DevMsg(eDLL_T::FS, "Mapping chunk '%zu' ('%s') to existing chunk at '0x%llx'\n", j, svEntryHash.c_str(), p.first->second.m_nPackFileOffset);
if (!jEntry.is_null())
{ vDescriptor = p.first->second;
iPreloadSize = jEntry.at("preloadSize").get<uint32_t>(); nSharedTotal += vDescriptor.m_nCompressedSize;
nLoadFlags = jEntry.at("loadFlags").get<uint32_t>(); nSharedCount++;
nTextureFlags = jEntry.at("textureFlags").get<uint16_t>();
bUseCompression = jEntry.at("useCompression").get<bool>(); continue;
bUseDataSharing = jEntry.at("useDataSharing").get<bool>();
}
}
catch (const std::exception& ex)
{
Warning(eDLL_T::FS, "Exception while reading VPK control file: '%s'\n", ex.what());
} }
} }
DevMsg(eDLL_T::FS, "Packing entry '%zu' ('%s')\n", i, svDestPath.c_str()); if (vKeys.m_bUseCompression)
vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), iPreloadSize, 0, nLoadFlags, nTextureFlags, svDestPath));
VPKEntryBlock_t& vEntry = vEntryBlocks[i];
for (size_t j = 0, es = vEntry.m_vFragments.size(); j < es; j++)
{ {
VPKChunkDescriptor_t& vDescriptor = vEntry.m_vFragments[j]; m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, s_EntryBuf, &vDescriptor.m_nCompressedSize, s_EntryBuf,
vDescriptor.m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
reader.Read(s_EntryBuf, vDescriptor.m_nUncompressedSize); if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
vDescriptor.m_nPackFileOffset = writer.GetPosition();
if (bUseDataSharing)
{ {
string svEntryHash = sha1(string(reinterpret_cast<char*>(s_EntryBuf), vDescriptor.m_nUncompressedSize)); Warning(eDLL_T::FS, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk packed without compression)\n",
auto p = m_mChunkHashMap.insert({ svEntryHash, vDescriptor }); m_lzCompStatus, j, i, vEntryBlocks[i].m_iPackFileIndex);
if (!p.second) // Map to existing chunk to avoid having copies of the same data.
{
DevMsg(eDLL_T::FS, "Mapping chunk '%zu' ('%s') to existing chunk at '0x%llx'\n", j, svEntryHash.c_str(), p.first->second.m_nPackFileOffset);
vDescriptor = p.first->second;
nSharedTotal += vDescriptor.m_nCompressedSize;
nSharedCount++;
continue;
}
}
if (bUseCompression)
{
m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, s_EntryBuf, &vDescriptor.m_nCompressedSize, s_EntryBuf,
vDescriptor.m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
{
Warning(eDLL_T::FS, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk packed without compression)\n",
m_lzCompStatus, j, i, vEntryBlocks[i].m_iPackFileIndex);
vDescriptor.m_nCompressedSize = vDescriptor.m_nUncompressedSize;
}
}
else // Write data uncompressed.
{
vDescriptor.m_nCompressedSize = vDescriptor.m_nUncompressedSize; vDescriptor.m_nCompressedSize = vDescriptor.m_nUncompressedSize;
} }
vDescriptor.m_bIsCompressed = vDescriptor.m_nCompressedSize != vDescriptor.m_nUncompressedSize;
writer.Write(s_EntryBuf, vDescriptor.m_nCompressedSize);
} }
else // Write data uncompressed.
{
vDescriptor.m_nCompressedSize = vDescriptor.m_nUncompressedSize;
}
vDescriptor.m_bIsCompressed = vDescriptor.m_nCompressedSize != vDescriptor.m_nUncompressedSize;
FileSystem()->Write(s_EntryBuf, vDescriptor.m_nCompressedSize, hPackFile);
} }
MemAllocSingleton()->Free(pBuf);
FileSystem()->Close(hAsset);
} }
DevMsg(eDLL_T::FS, "*** Build block totaling '%zu' bytes with '%zu' shared bytes among '%lu' chunks\n", writer.GetPosition(), nSharedTotal, nSharedCount);
DevMsg(eDLL_T::FS, "*** Build block totaling '%zu' bytes with '%zu' shared bytes among '%lu' chunks\n", FileSystem()->Tell(hPackFile), nSharedTotal, nSharedCount);
FileSystem()->Close(hPackFile);
m_mChunkHashMap.clear(); m_mChunkHashMap.clear();
memset(s_EntryBuf, '\0', sizeof(s_EntryBuf)); memset(s_EntryBuf, '\0', sizeof(s_EntryBuf));
@ -596,13 +626,21 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDir, const string& svWorkspa
Error(eDLL_T::FS, NO_ERROR, "Unsupported VPK directory file (invalid header criteria)\n"); Error(eDLL_T::FS, NO_ERROR, "Unsupported VPK directory file (invalid header criteria)\n");
return; return;
} }
BuildManifest(vDir.m_vEntryBlocks, svWorkspace, GetLevelName(vDir.m_svDirPath)); BuildManifest(vDir.m_vEntryBlocks, svWorkspace, GetLevelName(vDir.m_svDirPath));
const string svPath = RemoveFileName(vDir.m_svDirPath) + '/';
for (size_t i = 0, fs = vDir.m_vPackFile.size(); i < fs; i++) for (size_t i = 0, fs = vDir.m_vPackFile.size(); i < fs; i++)
{ {
const fs::path fspVpkPath(vDir.m_svDirPath); const string svPackFile = svPath + vDir.m_vPackFile[i];
const string svPath = fspVpkPath.parent_path().u8string() + '\\' + vDir.m_vPackFile[i];
CIOStream iStream(svPath, CIOStream::Mode_t::READ); // Create stream to read from each pack file. // Read from each pack file.
FileHandle_t hPackFile = FileSystem()->Open(svPackFile.c_str(), "rb", "GAME");
if (!hPackFile)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n", __FUNCTION__, svPackFile.c_str());
continue;
}
for (size_t j = 0, es = vDir.m_vEntryBlocks.size(); j < es; j++) for (size_t j = 0, es = vDir.m_vEntryBlocks.size(); j < es; j++)
{ {
@ -614,11 +652,11 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDir, const string& svWorkspa
else // Chunk belongs to this block. else // Chunk belongs to this block.
{ {
const string svFilePath = CreateDirectories(svWorkspace + vBlock.m_svEntryPath); const string svFilePath = CreateDirectories(svWorkspace + vBlock.m_svEntryPath);
CIOStream oStream(svFilePath, CIOStream::Mode_t::WRITE); FileHandle_t hAsset = FileSystem()->Open(svFilePath.c_str(), "wb", "GAME");
if (!oStream.IsWritable()) if (!hAsset)
{ {
Error(eDLL_T::FS, NO_ERROR, "Unable to write file '%s'\n", svFilePath.c_str()); Error(eDLL_T::FS, NO_ERROR, "%s - Unable to write to '%s' (read-only?)\n", __FUNCTION__, svFilePath.c_str());
continue; continue;
} }
@ -628,8 +666,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDir, const string& svWorkspa
const VPKChunkDescriptor_t& vChunk = vBlock.m_vFragments[k]; const VPKChunkDescriptor_t& vChunk = vBlock.m_vFragments[k];
m_nChunkCount++; m_nChunkCount++;
iStream.SetPosition(vChunk.m_nPackFileOffset); FileSystem()->Seek(hPackFile, vChunk.m_nPackFileOffset, FileSystemSeek_t::FILESYSTEM_SEEK_HEAD);
iStream.Read(s_EntryBuf, vChunk.m_nCompressedSize); FileSystem()->Read(s_EntryBuf, vChunk.m_nCompressedSize, hPackFile);
if (vChunk.m_bIsCompressed) if (vChunk.m_bIsCompressed)
{ {
@ -649,55 +687,80 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDir, const string& svWorkspa
} }
else // If successfully decompressed, write to file. else // If successfully decompressed, write to file.
{ {
oStream.Write(s_DecompBuf, nDstLen); FileSystem()->Write(s_DecompBuf, nDstLen, hAsset);
} }
} }
else // If not compressed, write source data into output file. else // If not compressed, write source data into output file.
{ {
oStream.Write(s_EntryBuf, vChunk.m_nUncompressedSize); FileSystem()->Write(s_EntryBuf, vChunk.m_nUncompressedSize, hAsset);
} }
} }
FileSystem()->Close(hAsset);
if (m_nChunkCount == vBlock.m_vFragments.size()) // Only validate after last entry in block had been written. if (m_nChunkCount == vBlock.m_vFragments.size()) // Only validate after last entry in block had been written.
{ {
m_nChunkCount = NULL; m_nChunkCount = NULL;
m_nCrc32_Internal = vBlock.m_nFileCRC; m_nCrc32_Internal = vBlock.m_nFileCRC;
oStream.Flush();
ValidateCRC32PostDecomp(svFilePath); ValidateCRC32PostDecomp(svFilePath);
} }
} }
} }
FileSystem()->Close(hPackFile);
} }
memset(s_EntryBuf, '\0', sizeof(s_EntryBuf)); memset(s_EntryBuf, '\0', sizeof(s_EntryBuf));
memset(s_DecompBuf, '\0', sizeof(s_DecompBuf)); memset(s_DecompBuf, '\0', sizeof(s_DecompBuf));
} }
//-----------------------------------------------------------------------------
// Purpose: 'VPKKeyValues_t' memory constructor
// Input : &svEntryPath -
// iPreloadSize -
// nLoadFlags -
// nTextureFlags -
// bUseCompression -
// bUseDataSharing -
//-----------------------------------------------------------------------------
VPKKeyValues_t::VPKKeyValues_t(const string& svEntryPath, uint16_t iPreloadSize, uint32_t nLoadFlags, uint16_t nTextureFlags, bool bUseCompression, bool bUseDataSharing)
{
m_svEntryPath = svEntryPath;
m_iPreloadSize = iPreloadSize;
m_nLoadFlags = nLoadFlags;
m_nTextureFlags = nTextureFlags;
m_bUseCompression = bUseCompression;
m_bUseDataSharing = bUseDataSharing;
}
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: 'VPKEntryBlock_t' file constructor // Purpose: 'VPKEntryBlock_t' file constructor
// Input : *pReader - // Input : hFile -
// svEntryPath - // svEntryPath -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* pReader, string svEntryPath) VPKEntryBlock_t::VPKEntryBlock_t(FileHandle_t hFile, string svEntryPath)
{ {
StringReplace(svEntryPath, "\\", "/"); // Flip windows-style backslash to forward slash. StringReplace(svEntryPath, "\\", "/"); // Flip windows-style backslash to forward slash.
StringReplace(svEntryPath, " /", "" ); // Remove space character representing VPK root. StringReplace(svEntryPath, " /", "" ); // Remove space character representing VPK root.
m_svEntryPath = svEntryPath; // Set the entry path. m_svEntryPath = svEntryPath; // Set the entry path.
pReader->Read<uint32_t>(m_nFileCRC); //
pReader->Read<uint16_t>(m_iPreloadSize); //
pReader->Read<uint16_t>(m_iPackFileIndex); //
FileSystem()->Read(&m_nFileCRC, sizeof(uint32_t), hFile); //
FileSystem()->Read(&m_iPreloadSize, sizeof(uint16_t), hFile); //
FileSystem()->Read(&m_iPackFileIndex, sizeof(uint16_t), hFile); //
uint16_t nMarker = 0;
do // Loop through all chunks in the entry and add to list. do // Loop through all chunks in the entry and add to list.
{ {
VPKChunkDescriptor_t entry(pReader); VPKChunkDescriptor_t entry(hFile);
m_vFragments.push_back(entry); m_vFragments.push_back(entry);
} while (pReader->Read<uint16_t>() != PACKFILEINDEX_END);
FileSystem()->Read(&nMarker, sizeof(nMarker), hFile);
} while (nMarker != static_cast<uint16_t>(PACKFILEINDEX_END));
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: 'VPKEntryBlock_t' memory constructor // Purpose: 'VPKEntryBlock_t' memory constructor
// Input : &vData - // Input : *pData -
// nOffset - // nOffset -
// iPreloadSize - // iPreloadSize -
// iPackFileIndex - // iPackFileIndex -
@ -705,16 +768,16 @@ VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* pReader, string svEntryPath)
// nTextureFlags - // nTextureFlags -
// &svEntryPath - // &svEntryPath -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKEntryBlock_t::VPKEntryBlock_t(const vector<uint8_t> &vData, int64_t nOffset, uint16_t iPreloadSize, VPKEntryBlock_t::VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize,
uint16_t iPackFileIndex, uint32_t nLoadFlags, uint16_t nTextureFlags, const string& svEntryPath) uint16_t iPackFileIndex, uint32_t nLoadFlags, uint16_t nTextureFlags, const string& svEntryPath)
{ {
m_nFileCRC = crc32::update(NULL, vData.data(), vData.size()); m_nFileCRC = crc32::update(NULL, pData, nLen);
m_iPreloadSize = iPreloadSize; m_iPreloadSize = iPreloadSize;
m_iPackFileIndex = iPackFileIndex; m_iPackFileIndex = iPackFileIndex;
m_svEntryPath = svEntryPath; m_svEntryPath = svEntryPath;
size_t nFragmentCount = (vData.size() + ENTRY_MAX_LEN - 1) / ENTRY_MAX_LEN; size_t nFragmentCount = (nLen + ENTRY_MAX_LEN - 1) / ENTRY_MAX_LEN;
size_t nFileSize = vData.size(); size_t nFileSize = nLen;
int64_t nCurrentOffset = nOffset; int64_t nCurrentOffset = nOffset;
for (size_t i = 0; i < nFragmentCount; i++) // Fragment data into 1 MiB chunks. for (size_t i = 0; i < nFragmentCount; i++) // Fragment data into 1 MiB chunks.
@ -728,15 +791,15 @@ VPKEntryBlock_t::VPKEntryBlock_t(const vector<uint8_t> &vData, int64_t nOffset,
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: 'VPKChunkDescriptor_t' file constructor // Purpose: 'VPKChunkDescriptor_t' file constructor
// Input : *pReader - // Input : hFile -
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKChunkDescriptor_t::VPKChunkDescriptor_t(CIOStream* pReader) VPKChunkDescriptor_t::VPKChunkDescriptor_t(FileHandle_t hFile)
{ {
pReader->Read<uint32_t>(m_nLoadFlags); // FileSystem()->Read(&m_nLoadFlags, sizeof(uint32_t), hFile); //
pReader->Read<uint16_t>(m_nTextureFlags); // FileSystem()->Read(&m_nTextureFlags, sizeof(uint16_t), hFile); //
pReader->Read<uint64_t>(m_nPackFileOffset); // FileSystem()->Read(&m_nPackFileOffset, sizeof(uint64_t), hFile); //
pReader->Read<uint64_t>(m_nCompressedSize); // FileSystem()->Read(&m_nCompressedSize, sizeof(uint64_t), hFile); //
pReader->Read<uint64_t>(m_nUncompressedSize); // FileSystem()->Read(&m_nUncompressedSize, sizeof(uint64_t), hFile); //
m_bIsCompressed = (m_nCompressedSize != m_nUncompressedSize); m_bIsCompressed = (m_nCompressedSize != m_nUncompressedSize);
} }
@ -765,17 +828,24 @@ VPKChunkDescriptor_t::VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextur
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
VPKDir_t::VPKDir_t(const string& svPath) VPKDir_t::VPKDir_t(const string& svPath)
{ {
CIOStream reader(svPath, CIOStream::Mode_t::READ); // Create stream to read from each pack file.
FileHandle_t hDirectory = FileSystem()->Open(svPath.c_str(), "rb", "GAME");
if (!hDirectory)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n", __FUNCTION__, svPath.c_str());
return;
}
reader.Read<uint32_t>(m_vHeader.m_nHeaderMarker); FileSystem()->Read(&m_vHeader.m_nHeaderMarker, sizeof(uint32_t), hDirectory);
reader.Read<uint16_t>(m_vHeader.m_nMajorVersion); // FileSystem()->Read(&m_vHeader.m_nMajorVersion, sizeof(uint16_t), hDirectory); //
reader.Read<uint16_t>(m_vHeader.m_nMinorVersion); // FileSystem()->Read(&m_vHeader.m_nMinorVersion, sizeof(uint16_t), hDirectory); //
reader.Read<uint32_t>(m_vHeader.m_nDirectorySize); // FileSystem()->Read(&m_vHeader.m_nDirectorySize, sizeof(uint32_t), hDirectory); //
reader.Read<uint32_t>(m_nFileDataSize); // FileSystem()->Read(&m_vHeader.m_nSignatureSize, sizeof(uint32_t), hDirectory); //
m_vEntryBlocks = g_pPackedStore->GetEntryBlocks(&reader); m_vEntryBlocks = g_pPackedStore->GetEntryBlocks(hDirectory);
m_svDirPath = svPath; // Set path to vpk directory file. m_svDirPath = svPath; // Set path to vpk directory file.
m_nPackFileCount = 0;
for (VPKEntryBlock_t vEntry : m_vEntryBlocks) for (VPKEntryBlock_t vEntry : m_vEntryBlocks)
{ {
if (vEntry.m_iPackFileIndex > m_nPackFileCount) if (vEntry.m_iPackFileIndex > m_nPackFileCount)
@ -798,15 +868,21 @@ VPKDir_t::VPKDir_t(const string& svPath)
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t>& vEntryBlocks) void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t>& vEntryBlocks)
{ {
CIOStream writer(svDirectoryFile, CIOStream::Mode_t::WRITE); FileHandle_t hDirectoryFile = FileSystem()->Open(svDirectoryFile.c_str(), "wb", "GAME");
if (!hDirectoryFile)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to write to '%s' (read-only?)\n", __FUNCTION__, svDirectoryFile.c_str());
return;
}
auto vMap = std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>(); auto vMap = std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>();
uint64_t nDescriptors = NULL; uint64_t nDescriptors = NULL;
writer.Write<uint32_t>(m_vHeader.m_nHeaderMarker); FileSystem()->Write(&m_vHeader.m_nHeaderMarker, sizeof(uint32_t), hDirectoryFile);
writer.Write<uint16_t>(m_vHeader.m_nMajorVersion); FileSystem()->Write(&m_vHeader.m_nMajorVersion, sizeof(uint16_t), hDirectoryFile);
writer.Write<uint16_t>(m_vHeader.m_nMinorVersion); FileSystem()->Write(&m_vHeader.m_nMinorVersion, sizeof(uint16_t), hDirectoryFile);
writer.Write<uint32_t>(m_vHeader.m_nDirectorySize); FileSystem()->Write(&m_vHeader.m_nDirectorySize, sizeof(uint32_t), hDirectoryFile);
writer.Write<uint32_t>(m_vHeader.m_nSignatureSize); FileSystem()->Write(&m_vHeader.m_nSignatureSize, sizeof(uint32_t), hDirectoryFile);
for (const VPKEntryBlock_t& vBlock : vEntryBlocks) for (const VPKEntryBlock_t& vBlock : vEntryBlocks)
{ {
@ -830,50 +906,53 @@ void VPKDir_t::Build(const string& svDirectoryFile, const vector<VPKEntryBlock_t
for (auto& iKeyValue : vMap) for (auto& iKeyValue : vMap)
{ {
writer.WriteString(iKeyValue.first); FileSystem()->Write(iKeyValue.first.c_str(), (iKeyValue.first.length() + 1), hDirectoryFile);
for (auto& jKeyValue : iKeyValue.second) for (auto& jKeyValue : iKeyValue.second)
{ {
writer.WriteString(jKeyValue.first); FileSystem()->Write(jKeyValue.first.c_str(), (jKeyValue.first.length() + 1), hDirectoryFile);
for (auto& vEntry : jKeyValue.second) for (auto& vEntry : jKeyValue.second)
{ {
/*Write entry block*/ string pszEntryPath = GetFileName(vEntry.m_svEntryPath, true);
writer.WriteString(GetFileName(vEntry.m_svEntryPath, true)); FileSystem()->Write(pszEntryPath.c_str(), (pszEntryPath.length() + 1), hDirectoryFile);
writer.Write(vEntry.m_nFileCRC);
writer.Write(vEntry.m_iPreloadSize); FileSystem()->Write(&vEntry.m_nFileCRC, sizeof(uint32_t), hDirectoryFile);
writer.Write(vEntry.m_iPackFileIndex); FileSystem()->Write(&vEntry.m_iPreloadSize, sizeof(uint16_t), hDirectoryFile);
FileSystem()->Write(&vEntry.m_iPackFileIndex, sizeof(uint16_t), hDirectoryFile);
for (size_t i = 0, nc = vEntry.m_vFragments.size(); i < nc; i++) for (size_t i = 0, nc = vEntry.m_vFragments.size(); i < nc; i++)
{ {
/*Write chunk descriptor*/ /*Write chunk descriptor*/
const VPKChunkDescriptor_t* pDescriptor = &vEntry.m_vFragments[i]; const VPKChunkDescriptor_t* pDescriptor = &vEntry.m_vFragments[i];
writer.Write(pDescriptor->m_nLoadFlags); FileSystem()->Write(&pDescriptor->m_nLoadFlags, sizeof(uint32_t), hDirectoryFile);
writer.Write(pDescriptor->m_nTextureFlags); FileSystem()->Write(&pDescriptor->m_nTextureFlags, sizeof(uint16_t), hDirectoryFile);
writer.Write(pDescriptor->m_nPackFileOffset); FileSystem()->Write(&pDescriptor->m_nPackFileOffset, sizeof(uint64_t), hDirectoryFile);
writer.Write(pDescriptor->m_nCompressedSize); FileSystem()->Write(&pDescriptor->m_nCompressedSize, sizeof(uint64_t), hDirectoryFile);
writer.Write(pDescriptor->m_nUncompressedSize); FileSystem()->Write(&pDescriptor->m_nUncompressedSize, sizeof(uint64_t), hDirectoryFile);
if (i != (nc - 1)) if (i != (nc - 1))
{ {
writer.Write<uint16_t>(NULL); FileSystem()->Write(&PACKFILEINDEX_SEP, sizeof(uint16_t), hDirectoryFile);
} }
else // Mark end of entry. else // Mark end of entry.
{ {
writer.Write<uint16_t>(PACKFILEINDEX_END); FileSystem()->Write(&PACKFILEINDEX_END, sizeof(uint16_t), hDirectoryFile);
} }
nDescriptors++; nDescriptors++;
} }
} }
writer.Write<uint8_t>('\0'); FileSystem()->Write(&PACKFILEINDEX_SEP, sizeof(uint8_t), hDirectoryFile);
} }
writer.Write<uint8_t>('\0'); FileSystem()->Write(&PACKFILEINDEX_SEP, sizeof(uint8_t), hDirectoryFile);
} }
writer.Write<uint8_t>('\0'); FileSystem()->Write(&PACKFILEINDEX_SEP, sizeof(uint8_t), hDirectoryFile);
m_vHeader.m_nDirectorySize = static_cast<uint32_t>(writer.GetPosition() - sizeof(VPKDirHeader_t)); m_vHeader.m_nDirectorySize = static_cast<uint32_t>(FileSystem()->Tell(hDirectoryFile) - sizeof(VPKDirHeader_t));
writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nDirectorySize)); FileSystem()->Seek(hDirectoryFile, offsetof(VPKDir_t, m_vHeader.m_nDirectorySize), FileSystemSeek_t::FILESYSTEM_SEEK_HEAD);
writer.Write(m_vHeader.m_nDirectorySize); FileSystem()->Write(&m_vHeader.m_nDirectorySize, sizeof(uint32_t), hDirectoryFile);
writer.Write(0); FileSystem()->Write(&PACKFILEINDEX_SEP, sizeof(uint32_t), hDirectoryFile);
FileSystem()->Close(hDirectoryFile);
DevMsg(eDLL_T::FS, "*** Build directory totaling '%zu' bytes with '%zu' entries and '%zu' descriptors\n", DevMsg(eDLL_T::FS, "*** Build directory totaling '%zu' bytes with '%zu' entries and '%zu' descriptors\n",
size_t(sizeof(VPKDirHeader_t) + m_vHeader.m_nDirectorySize), vEntryBlocks.size(), nDescriptors); size_t(sizeof(VPKDirHeader_t) + m_vHeader.m_nDirectorySize), vEntryBlocks.size(), nDescriptors);

81
r5dev/vpklib/packedstore.h
View File

@ -8,7 +8,7 @@
* ██║ ██║ ██║ ╚████╔╝ ██║ ██║ ██╗ ███████╗██║██████╔╝ * * ██║ ██║ ██║ ╚████╔╝ ██║ ██║ ██╗ ███████╗██║██████╔╝ *
* ╚═╝ ╚═╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝╚═════╝ * * ╚═╝ ╚═╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝╚═════╝ *
*******************************************************************/ *******************************************************************/
#include "public/utility/binstream.h" #include "public/ifilesystem.h"
#include "thirdparty/lzham/include/lzham.h" #include "thirdparty/lzham/include/lzham.h"
constexpr unsigned int VPK_HEADER_MARKER = 0x55AA1234; constexpr unsigned int VPK_HEADER_MARKER = 0x55AA1234;
@ -17,6 +17,7 @@ constexpr unsigned int VPK_MINOR_VERSION = 3;
constexpr unsigned int VPK_DICT_SIZE = 20; constexpr unsigned int VPK_DICT_SIZE = 20;
constexpr int ENTRY_MAX_LEN = 1024 * 1024; constexpr int ENTRY_MAX_LEN = 1024 * 1024;
constexpr int PACKFILEPATCH_MAX = 512; constexpr int PACKFILEPATCH_MAX = 512;
constexpr int PACKFILEINDEX_SEP = 0x0;
constexpr int PACKFILEINDEX_END = 0xffff; constexpr int PACKFILEINDEX_END = 0xffff;
static const std::regex BLOCK_REGEX{ R"(pak000_([0-9]{3}))" }; static const std::regex BLOCK_REGEX{ R"(pak000_([0-9]{3}))" };
@ -42,56 +43,22 @@ static const vector<string> DIR_LOCALE =
"tchinese" "tchinese"
}; };
enum class EPackedLoadFlags : int struct VPKKeyValues_t
{ {
LOAD_NONE, static constexpr uint16_t TEXTURE_FLAGS_DEFAULT = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
LOAD_VISIBLE = 1 << 0, // Visible to FileSystem. static constexpr uint32_t LOAD_FLAGS_DEFAULT = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE);
LOAD_CACHE = 1 << 8, // Only set for assets not stored in the depot directory.
LOAD_TEXTURE_UNK0 = 1 << 18,
LOAD_TEXTURE_UNK1 = 1 << 19,
LOAD_TEXTURE_UNK2 = 1 << 20,
};
enum class EPackedTextureFlags : short string m_svEntryPath;
{ uint16_t m_iPreloadSize;
TEXTURE_NONE, uint32_t m_nLoadFlags;
TEXTURE_DEFAULT = 1 << 3, uint16_t m_nTextureFlags;
TEXTURE_ENVIRONMENT_MAP = 1 << 10, bool m_bUseCompression;
}; bool m_bUseDataSharing;
struct FileHandleTracker_t VPKKeyValues_t(const string& svEntryPath = "", uint16_t iPreloadSize = NULL, uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT,
{ uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT, bool bUseCompression = true, bool bUseDataSharing = true);
int m_nFileNumber;
int m_nCurOfs;
HANDLE m_hFileHandle;
}; };
struct pFileHandleTracker_t
{
FileHandleTracker_t self[1024];
};
#pragma pack(push, 1)
struct VPKFileEntry_t
{
char* m_pszDirectory;
char* m_pszFileName;
char* m_pszExtension;
uint8_t unknown[0x38];
};
struct VPKData_t
{
int m_nHandle;
char pad[1];
char m_szPath[255];
uint8_t unknown2[0x134];
int32_t m_nEntries;
uint8_t unknown3[12];
VPKFileEntry_t* m_pEntries;
};
#pragma pack(pop)
struct VPKChunkDescriptor_t struct VPKChunkDescriptor_t
{ {
uint32_t m_nLoadFlags; // Load flags. uint32_t m_nLoadFlags; // Load flags.
@ -102,7 +69,7 @@ struct VPKChunkDescriptor_t
bool m_bIsCompressed = false; bool m_bIsCompressed = false;
VPKChunkDescriptor_t(){}; VPKChunkDescriptor_t(){};
VPKChunkDescriptor_t(CIOStream* pReader); VPKChunkDescriptor_t(FileHandle_t pFile);
VPKChunkDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize); VPKChunkDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize);
}; };
@ -114,8 +81,9 @@ struct VPKEntryBlock_t
vector<VPKChunkDescriptor_t> m_vFragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks). vector<VPKChunkDescriptor_t> m_vFragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks).
string m_svEntryPath; // Path to entry within vpk. string m_svEntryPath; // Path to entry within vpk.
VPKEntryBlock_t(CIOStream* pReader, string svEntryPath); VPKEntryBlock_t(FileHandle_t pFile, string svEntryPath);
VPKEntryBlock_t(const vector<uint8_t>& vData, int64_t nOffset, uint16_t iPreloadSize, uint16_t iPackFileIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, const string& svEntryPath); VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize,
uint16_t iPackFileIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, const string& svEntryPath);
}; };
struct VPKDirHeader_t struct VPKDirHeader_t
@ -130,7 +98,6 @@ struct VPKDirHeader_t
struct VPKDir_t struct VPKDir_t
{ {
VPKDirHeader_t m_vHeader; // Dir header. VPKDirHeader_t m_vHeader; // Dir header.
uint32_t m_nFileDataSize; // File data section size.
vector<VPKEntryBlock_t> m_vEntryBlocks; // Vector of entry blocks. vector<VPKEntryBlock_t> m_vEntryBlocks; // Vector of entry blocks.
uint16_t m_nPackFileCount; // Number of pack patches (pack file count-1). uint16_t m_nPackFileCount; // Number of pack patches (pack file count-1).
vector<string> m_vPackFile; // Vector of pack file names. vector<string> m_vPackFile; // Vector of pack file names.
@ -158,17 +125,17 @@ public:
string GetPackFile(const string& svPackDirFile, uint16_t iPackFileIndex) const; string GetPackFile(const string& svPackDirFile, uint16_t iPackFileIndex) const;
lzham_compress_level GetCompressionLevel(void) const; lzham_compress_level GetCompressionLevel(void) const;
vector<VPKEntryBlock_t> GetEntryBlocks(CIOStream* pReader) const; vector<VPKEntryBlock_t> GetEntryBlocks(FileHandle_t pDirectory) const;
vector<string> GetEntryPaths(const string& svPathIn) const; vector<VPKKeyValues_t> GetEntryPaths(const string& svPathIn) const;
vector<string> GetEntryPaths(const string& svPathIn, const nlohmann::json& jManifest) const; vector<VPKKeyValues_t> GetEntryPaths(const string& svPathIn, KeyValues* pManifestKeyValues) const;
string GetNameParts(const string& svDirectoryName, int nCaptureGroup) const; string GetNameParts(const string& svDirectoryName, int nCaptureGroup) const;
string GetLevelName(const string& svDirectoryName) const; string GetLevelName(const string& svDirectoryName) const;
nlohmann::json GetManifest(const string& svWorkspace, const string& svManifestName) const; KeyValues* GetManifest(const string& svWorkspace, const string& svManifestName) const;
vector<string> GetIgnoreList(const string& svWorkspace) const; vector<string> GetIgnoreList(const string& svWorkspace) const;
string FormatEntryPath(string svName, const string& svPath, const string& svExtension) const; string FormatEntryPath(const string& svName, const string& svPath, const string& svExtension) const;
string StripLocalePrefix(const string& svDirectoryFile) const; string StripLocalePrefix(const string& svDirectoryFile) const;
VPKPair_t BuildFileName(string svLanguage, string svTarget, const string& svLevel, int nPatch) const; VPKPair_t BuildFileName(string svLanguage, string svTarget, const string& svLevel, int nPatch) const;
@ -177,8 +144,8 @@ public:
void PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly); void PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly);
void UnpackWorkspace(const VPKDir_t& vDir, const string& svPathOut = ""); void UnpackWorkspace(const VPKDir_t& vDir, const string& svPathOut = "");
void ValidateAdler32PostDecomp(const string& svDirAsset); void ValidateAdler32PostDecomp(const string& svAssetPath);
void ValidateCRC32PostDecomp(const string& svDirAsset); void ValidateCRC32PostDecomp(const string& svAssetPath);
private: private:
size_t m_nChunkCount; // The number of fragments for this asset. size_t m_nChunkCount; // The number of fragments for this asset.

3
r5dev/vstdlib/callback.cpp
View File

@ -55,6 +55,7 @@
#ifndef DEDICATED #ifndef DEDICATED
#include "game/client/viewrender.h" #include "game/client/viewrender.h"
#endif // !DEDICATED #endif // !DEDICATED
#include "public/utility/binstream.h"
/* /*
@ -569,7 +570,7 @@ void VPK_Unpack_f(const CCommand& args)
timer.Start(); timer.Start();
g_pPackedStore->InitLzDecompParams(); g_pPackedStore->InitLzDecompParams();
g_pPackedStore->UnpackWorkspace(vpk, ConvertToWinPath(fs_packedstore_workspace->GetString())); g_pPackedStore->UnpackWorkspace(vpk, fs_packedstore_workspace->GetString());
timer.End(); timer.End();
DevMsg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds()); DevMsg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds());