R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

CPackedStore refactor

Browse Source 
* Improve naming convention, and make it more consistent.
* Use engine types as much as possible.
* Reduced the amount of string copy constructions.
This commit is contained in:
Kawe Mazidjatari 2023-05-29 21:41:17 +02:00
parent 7f15b94cd9
commit abef34d3ef
6 changed files with 531 additions and 430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
r5dev/common/callback.cpp
View File

@ -606,11 +606,11 @@ void VPK_Pack_f(const CCommand& args)
VPKPair_t pair(args.Arg(1), args.Arg(2), args.Arg(3), NULL);
CFastTimer timer;
DevMsg(eDLL_T::FS, "*** Starting VPK build command for: '%s'\n", pair.m_svDirectoryName.c_str());
DevMsg(eDLL_T::FS, "*** Starting VPK build command for: '%s'\n", pair.m_DirName.Get());
timer.Start();
g_pPackedStore->InitLzCompParams();
g_pPackedStore->PackWorkspace(pair, fs_packedstore_workspace->GetString(), "vpk/", (args.ArgC() > 4));
g_pPackedStore->PackWorkspace(pair, fs_packedstore_workspace->GetString(), "vpk/");
timer.End();
DevMsg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds());
@ -632,11 +632,11 @@ void VPK_Unpack_f(const CCommand& args)
return;
}
const char* pArg = args.Arg(1);
VPKDir_t vpk(pArg, (args.ArgC() > 2));
CUtlString arg = args.Arg(1);
VPKDir_t vpk(arg, (args.ArgC() > 2));
CFastTimer timer;
DevMsg(eDLL_T::FS, "*** Starting VPK extraction command for: '%s'\n", pArg);
DevMsg(eDLL_T::FS, "*** Starting VPK extraction command for: '%s'\n", arg.Get());
timer.Start();
g_pPackedStore->InitLzDecompParams();

2
r5dev/common/global.cpp
View File

@ -377,7 +377,7 @@ void ConVar_StaticInit(void)
// FILESYSTEM |
fs_showWarnings = ConVar::StaticCreate("fs_showWarnings" , "0", FCVAR_DEVELOPMENTONLY, "Logs the FileSystem warnings to the console, filtered by 'fs_warning_level' ( !slower! ).", true, 0.f, true, 2.f, nullptr, "0 = log to file. 1 = 0 + log to console. 2 = 1 + log to notify.");
fs_packedstore_entryblock_stats = ConVar::StaticCreate("fs_packedstore_entryblock_stats" , "0", FCVAR_DEVELOPMENTONLY, "Logs the stats of each file entry in the VPK during decompression ( !slower! ).", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_workspace = ConVar::StaticCreate("fs_packedstore_workspace" , "platform/ship/", FCVAR_DEVELOPMENTONLY, "Determines the current VPK workspace.", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_workspace = ConVar::StaticCreate("fs_packedstore_workspace" , "ship", FCVAR_DEVELOPMENTONLY, "Determines the current VPK workspace.", false, 0.f, false, 0.f, nullptr, nullptr);
fs_packedstore_compression_level = ConVar::StaticCreate("fs_packedstore_compression_level", "default", FCVAR_DEVELOPMENTONLY, "Determines the VPK compression level.", false, 0.f, false, 0.f, nullptr, "fastest faster default better uber");
fs_packedstore_max_helper_threads = ConVar::StaticCreate("fs_packedstore_max_helper_threads" , "-1", FCVAR_DEVELOPMENTONLY, "Max # of additional \"helper\" threads to create during compression.", true, -1, true, LZHAM_MAX_HELPER_THREADS, nullptr, "Must range between [-1,LZHAM_MAX_HELPER_THREADS], where -1=max practical.");
//-------------------------------------------------------------------------

8
r5dev/filesystem/basefilesystem.cpp
View File

@ -190,9 +190,9 @@ const char* CBaseFileSystem::VUnmountVPKFile(CBaseFileSystem* pFileSystem, const
// Input : *pFile -
// Output : string
//---------------------------------------------------------------------------------
string CBaseFileSystem::ReadString(FileHandle_t pFile)
CUtlString CBaseFileSystem::ReadString(FileHandle_t pFile)
{
string svString;
CUtlString result;
char c = '\0';
do
@ -200,11 +200,11 @@ string CBaseFileSystem::ReadString(FileHandle_t pFile)
Read(&c, sizeof(char), pFile);
if (c)
svString += c;
result += c;
} while (c);
return svString;
return result;
}
void VBaseFileSystem::Attach() const

2
r5dev/filesystem/basefilesystem.h
View File

@ -15,7 +15,7 @@ public:
static VPKData_t* VMountVPKFile(CBaseFileSystem* pFileSystem, const char* pszVpkPath);
static const char* VUnmountVPKFile(CBaseFileSystem* pFileSystem, const char* pszVpkPath);
string ReadString(FileHandle_t pFile);
CUtlString ReadString(FileHandle_t pFile);
protected:
//----------------------------------------------------------------------------

825
r5dev/vpklib/packedstore.cpp
View File

File diff suppressed because it is too large Load Diff

114
r5dev/vpklib/packedstore.h
View File

@ -9,6 +9,9 @@
* ╚═╝ ╚═╝ ╚═╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝╚═════╝ *
*******************************************************************/
#include "public/ifilesystem.h"
#include "public/tier1/strtools.h"
#include "public/tier1/utlvector.h"
#include "public/tier1/utlstring.h"
#include "thirdparty/lzham/include/lzham.h"
constexpr unsigned int VPK_HEADER_MARKER = 0x55AA1234;
@ -19,16 +22,17 @@ constexpr int ENTRY_MAX_LEN = 1024 * 1024;
constexpr int PACKFILEPATCH_MAX = 512;
constexpr int PACKFILEINDEX_SEP = 0x0;
constexpr int PACKFILEINDEX_END = 0xffff;
constexpr const char VPK_IGNORE_FILE[] = ".vpkignore";
static const std::regex BLOCK_REGEX{ R"(pak000_([0-9]{3}))" };
static const std::regex DIR_REGEX{ R"((?:.*\/)?([^_]*_)(.*)(.bsp.pak000_dir).*)" };
static const vector<const char*> DIR_TARGET =
static const char* const DIR_TARGET[]
{
"server",
"client"
};
static const vector<const char*> DIR_LOCALE =
static const char* const DIR_LOCALE[]
{
"english",
"french",
@ -48,14 +52,14 @@ struct VPKKeyValues_t
static constexpr uint16_t TEXTURE_FLAGS_DEFAULT = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
static constexpr uint32_t LOAD_FLAGS_DEFAULT = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE);
string m_svEntryPath;
CUtlString m_EntryPath;
uint16_t m_iPreloadSize;
uint32_t m_nLoadFlags;
uint16_t m_nTextureFlags;
bool m_bUseCompression;
bool m_bDeduplicate;
VPKKeyValues_t(const string& svEntryPath = "", uint16_t iPreloadSize = NULL, uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT,
VPKKeyValues_t(const CUtlString& svEntryPath = "", uint16_t iPreloadSize = NULL, uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT,
uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT, bool bUseCompression = true, bool bDeduplicate = true);
};
@ -67,21 +71,39 @@ struct VPKChunkDescriptor_t
uint64_t m_nCompressedSize; // Compressed size of chunk.
uint64_t m_nUncompressedSize; // Uncompressed size of chunk.
VPKChunkDescriptor_t()
: m_nLoadFlags(0)
, m_nTextureFlags(0)
, m_nPackFileOffset(0)
, m_nCompressedSize(0)
, m_nUncompressedSize(0)
{
}
VPKChunkDescriptor_t(FileHandle_t hDirectoryFile);
VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize);
};
struct VPKEntryBlock_t
{
uint32_t m_nFileCRC; // Crc32 for the uncompressed entry.
uint16_t m_iPreloadSize; // Preload bytes.
uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry.
vector<VPKChunkDescriptor_t> m_vFragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks).
string m_svEntryPath; // Path to entry within vpk.
uint32_t m_nFileCRC; // Crc32 for the uncompressed entry.
uint16_t m_iPreloadSize; // Preload bytes.
uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry.
CUtlVector<VPKChunkDescriptor_t> m_Fragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks).
CUtlString m_EntryPath; // Path to entry within vpk.
VPKEntryBlock_t(FileHandle_t pFile, const string& svEntryPath);
VPKEntryBlock_t(FileHandle_t pFile, const char* svEntryPath);
VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize,
uint16_t iPackFileIndex, uint32_t nLoadFlags, uint16_t nTextureFlags, const string& svEntryPath);
uint16_t iPackFileIndex, uint32_t nLoadFlags, uint16_t nTextureFlags, const char* svEntryPath);
VPKEntryBlock_t(const VPKEntryBlock_t& other)
: m_nFileCRC(other.m_nFileCRC)
, m_iPreloadSize(other.m_iPreloadSize)
, m_iPackFileIndex(other.m_iPackFileIndex)
, m_EntryPath(other.m_EntryPath)
{
// Has to be explicitly copied!
m_Fragments = other.m_Fragments;
}
};
struct VPKDirHeader_t
@ -95,40 +117,40 @@ struct VPKDirHeader_t
struct VPKPair_t
{
string m_svPackName;
string m_svDirectoryName;
CUtlString m_PackName;
CUtlString m_DirName;
VPKPair_t(const string& svLocale, const string& svTarget, const string& svLevel, int nPatch);
VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch);
};
struct VPKDir_t
{
VPKDirHeader_t m_vHeader; // Dir header.
vector<VPKEntryBlock_t> m_vEntryBlocks; // Vector of entry blocks.
uint16_t m_nPackFileCount; // Number of pack patches (pack file count-1).
vector<string> m_vPackFile; // Vector of pack file names.
string m_svDirectoryPath;// Path to vpk_dir file.
VPKDirHeader_t m_Header; // Dir header.
CUtlVector<VPKEntryBlock_t> m_EntryBlocks; // Vector of entry blocks.
uint16_t m_PackFileCount; // Number of pack patches (pack file count-1).
CUtlVector<CUtlString> m_PackFiles; // Vector of pack file names.
CUtlString m_DirFilePath; // Path to vpk_dir file.
VPKDir_t()
{
m_vHeader.m_nHeaderMarker = VPK_HEADER_MARKER; m_vHeader.m_nMajorVersion = VPK_MAJOR_VERSION;
m_vHeader.m_nMinorVersion = VPK_MINOR_VERSION; m_vHeader.m_nDirectorySize = NULL, m_vHeader.m_nSignatureSize = NULL;
m_nPackFileCount = NULL;
m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION;
m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL;
m_PackFileCount = NULL;
};
VPKDir_t(const string& svDirectoryFile);
VPKDir_t(const string& svDirectoryFile, bool bSanitizeName);
VPKDir_t(const CUtlString& svDirectoryFile);
VPKDir_t(const CUtlString& svDirectoryFile, bool bSanitizeName);
void Init(const string& svPath);
void Init(const CUtlString& svPath);
string StripLocalePrefix(const string& svDirectoryFile) const;
string GetPackFile(const string& svDirectoryPath, uint16_t iPackFileIndex) const;
CUtlString StripLocalePrefix(const CUtlString& svDirectoryFile) const;
CUtlString GetPackFile(const CUtlString& svDirectoryPath, uint16_t iPackFileIndex) const;
void WriteHeader(FileHandle_t hDirectoryFile) const;
void WriteTreeSize(FileHandle_t hDirectoryFile) const;
uint64_t WriteDescriptor(FileHandle_t hDirectoryFile, std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>& vMap) const;
uint64_t WriteDescriptor(FileHandle_t hDirectoryFile, std::map<CUtlString, std::map<CUtlString, std::list<VPKEntryBlock_t>>>& vMap) const;
void BuildDirectoryTree(const vector<VPKEntryBlock_t>& vEntryBlocks, std::map<string, std::map<string, std::list<VPKEntryBlock_t>>>& vMap) const;
void BuildDirectoryFile(const string& svDirectoryFile, const vector<VPKEntryBlock_t>& vEntryBlocks);
void BuildDirectoryTree(const CUtlVector<VPKEntryBlock_t>& vEntryBlocks, std::map<CUtlString, std::map<CUtlString, std::list<VPKEntryBlock_t>>>& vMap) const;
void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector<VPKEntryBlock_t>& vEntryBlocks);
};
class CPackedStore
@ -139,30 +161,30 @@ public:
lzham_compress_level GetCompressionLevel(void) const;
vector<VPKEntryBlock_t> GetEntryBlocks(FileHandle_t hDirectoryFile) const;
vector<VPKKeyValues_t> GetEntryValues(const string& svWorkspace) const;
vector<VPKKeyValues_t> GetEntryValues(const string& svWorkspace, KeyValues* pManifestKeyValues) const;
void GetEntryBlocks(CUtlVector<VPKEntryBlock_t>& entryBlocks, FileHandle_t hDirectoryFile) const;
bool GetEntryValues(CUtlVector<VPKKeyValues_t>& entryValues, const CUtlString& workspacePath, const CUtlString& dirFileName) const;
string GetNameParts(const string& svDirectoryName, int nCaptureGroup) const;
string GetLevelName(const string& svDirectoryName) const;
CUtlString GetNameParts(const CUtlString& dirFileName, int nCaptureGroup) const;
CUtlString GetLevelName(const CUtlString& dirFileName) const;
KeyValues* GetManifest(const string& svWorkspace, const string& svManifestName) const;
vector<string> GetIgnoreList(const string& svWorkspace) const;
KeyValues* GetManifest(const CUtlString& workspacePath, const CUtlString& manifestFile) const;
bool GetIgnoreList(CUtlVector<CUtlString>& ignoreList, const CUtlString& workspacePath) const;
string FormatEntryPath(const string& svPath, const string& svName, const string& svExtension) const;
void BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const string& svWorkspace, const string& svManifestName) const;
CUtlString FormatEntryPath(const CUtlString& filePath, const CUtlString& fileName, const CUtlString& fileExt) const;
void BuildManifest(const CUtlVector<VPKEntryBlock_t>& entryBlocks, const CUtlString& workspacePath, const CUtlString& manifestName) const;
void ValidateCRC32PostDecomp(const string& svAssetPath, const uint32_t nFileCRC);
void ValidateCRC32PostDecomp(const CUtlString& assetPath, const uint32_t nFileCRC);
bool Deduplicate(const uint8_t* pEntryBuffer, VPKChunkDescriptor_t& descriptor, const size_t chunkIndex);
void PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly);
void UnpackWorkspace(const VPKDir_t& vDirectory, const string& svWorkspace = "");
bool ShouldPrune(const CUtlString& filePath, CUtlVector<CUtlString>& ignoreList) const;
void PackWorkspace(const VPKPair_t& vpkPair, const char* workspaceName, const char* buildPath);
void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = "");
private:
size_t m_nChunkCount; // The number of fragments for this asset.
lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
std::unordered_map<string, const VPKChunkDescriptor_t&> m_mChunkHashMap;
lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap;
};
///////////////////////////////////////////////////////////////////////////////
extern CPackedStore* g_pPackedStore;