R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

VPK utility code improvements

Browse Source 
- Only process referenced pack files, previously, if an entry in the VPK file was build into a pack of index 128, the code would generate a list of pack files from 0 to 128.
- Added documentation to VPK structure.
This commit is contained in:
Kawe Mazidjatari 2023-07-06 00:49:17 +02:00
parent 5939e05331
commit c215fcc171
3 changed files with 99 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
r5dev/public/ipackedstore.h
View File

@ -1,7 +1,7 @@
#ifndef IPACKEDSTORE_H #ifndef IPACKEDSTORE_H
#define IPACKEDSTORE_H #define IPACKEDSTORE_H
enum class EPackedLoadFlags : int enum EPackedLoadFlags
{ {
LOAD_NONE, LOAD_NONE,
LOAD_VISIBLE = 1 << 0, // Visible to FileSystem. LOAD_VISIBLE = 1 << 0, // Visible to FileSystem.
@ -11,7 +11,7 @@ enum class EPackedLoadFlags : int
LOAD_TEXTURE_UNK2 = 1 << 20, LOAD_TEXTURE_UNK2 = 1 << 20,
}; };
enum class EPackedTextureFlags : short enum EPackedTextureFlags
{ {
TEXTURE_NONE, TEXTURE_NONE,
TEXTURE_DEFAULT = 1 << 3, TEXTURE_DEFAULT = 1 << 3,

36
r5dev/vpklib/packedstore.cpp
View File

@ -582,9 +582,9 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
BuildManifest(vpkDir.m_EntryBlocks, workspacePath, GetLevelName(vpkDir.m_DirFilePath)); BuildManifest(vpkDir.m_EntryBlocks, workspacePath, GetLevelName(vpkDir.m_DirFilePath));
const CUtlString basePath = vpkDir.m_DirFilePath.StripFilename(false); const CUtlString basePath = vpkDir.m_DirFilePath.StripFilename(false);
FOR_EACH_VEC(vpkDir.m_PackFiles, i) for (uint16_t packFileIndex : vpkDir.m_PakFileIndices)
{ {
const CUtlString packFile = basePath + vpkDir.m_PackFiles[i]; const CUtlString packFile = basePath + vpkDir.GetPackFileNameForIndex(packFileIndex);
// Read from each pack file. // Read from each pack file.
FileHandle_t hPackFile = FileSystem()->Open(packFile.Get(), "rb", "GAME"); FileHandle_t hPackFile = FileSystem()->Open(packFile.Get(), "rb", "GAME");
@ -598,7 +598,7 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
{ {
const VPKEntryBlock_t& entryBlock = vpkDir.m_EntryBlocks[j]; const VPKEntryBlock_t& entryBlock = vpkDir.m_EntryBlocks[j];
if (entryBlock.m_iPackFileIndex != uint16_t(i)) if (entryBlock.m_iPackFileIndex != packFileIndex)
{ {
// Chunk doesn't belongs to this block. // Chunk doesn't belongs to this block.
continue; continue;
@ -616,7 +616,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
continue; continue;
} }
DevMsg(eDLL_T::FS, "Unpacking entry '%i' from block '%i' ('%s')\n", j, i, entryBlock.m_EntryPath.Get()); DevMsg(eDLL_T::FS, "Unpacking entry '%i' from block '%i' ('%s')\n",
j, entryBlock.m_iPackFileIndex, entryBlock.m_EntryPath.Get());
FOR_EACH_VEC(entryBlock.m_Fragments, k) FOR_EACH_VEC(entryBlock.m_Fragments, k)
{ {
@ -642,8 +643,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS) if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
{ {
Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk not decompressed)\n", Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%i' within entry '%i' in block '%hu' (chunk not decompressed)\n",
lzDecompStatus, k, j, i); lzDecompStatus, k, j, packFileIndex);
} }
else // If successfully decompressed, write to file. else // If successfully decompressed, write to file.
{ {
@ -906,37 +907,26 @@ void VPKDir_t::Init(const CUtlString& dirFilePath)
FileSystem()->Read(&m_Header.m_nSignatureSize, sizeof(uint32_t), hDirFile); // FileSystem()->Read(&m_Header.m_nSignatureSize, sizeof(uint32_t), hDirFile); //
g_pPackedStore->GetEntryBlocks(m_EntryBlocks, hDirFile); g_pPackedStore->GetEntryBlocks(m_EntryBlocks, hDirFile);
m_DirFilePath = dirFilePath; // Set path to vpk directory file. m_DirFilePath = dirFilePath; // Set path to vpk directory file.
m_PackFileCount = 0;
// Obtain every referenced pack file from the directory tree.
FOR_EACH_VEC(m_EntryBlocks, i) FOR_EACH_VEC(m_EntryBlocks, i)
{ {
const VPKEntryBlock_t& entryBlock = m_EntryBlocks[i]; const VPKEntryBlock_t& entryBlock = m_EntryBlocks[i];
m_PakFileIndices.insert(entryBlock.m_iPackFileIndex);
if (entryBlock.m_iPackFileIndex > m_PackFileCount)
{
m_PackFileCount = entryBlock.m_iPackFileIndex;
}
}
for (uint16_t i = 0; i < m_PackFileCount + 1; i++)
{
m_PackFiles.AddToTail(GetPackFile(dirFilePath, i));
} }
FileSystem()->Close(hDirFile); FileSystem()->Close(hDirFile);
} }
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
// Purpose: formats pack file path for specific directory file // Purpose: formats pack file path for specified patch
// Input : &directoryPath - // Input : iPackFileIndex - (patch)
// iPackFileIndex -
// output : string // output : string
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
CUtlString VPKDir_t::GetPackFile(const CUtlString& directoryPath, uint16_t iPackFileIndex) const CUtlString VPKDir_t::GetPackFileNameForIndex(uint16_t iPackFileIndex) const
{ {
CUtlString packChunkName = StripLocalePrefix(directoryPath); CUtlString packChunkName = StripLocalePrefix(m_DirFilePath);
CUtlString packChunkIndex; CUtlString packChunkIndex;
packChunkIndex.Format("pak000_%03d", iPackFileIndex); packChunkIndex.Format("pak000_%03d", iPackFileIndex);

122
r5dev/vpklib/packedstore.h
View File

@ -44,10 +44,18 @@ static const char* const DIR_LOCALE[]
"tchinese" "tchinese"
}; };
//-----------------------------------------------------------------------------
// KeyValues structure for the VPK manifest file. This struct gets populated by
// the VPK's corresponding manifest file, which ultimately determines how each
// asset is getting packed into the VPK.
//-----------------------------------------------------------------------------
struct VPKKeyValues_t struct VPKKeyValues_t
{ {
static constexpr uint16_t TEXTURE_FLAGS_DEFAULT = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT); static constexpr uint16_t TEXTURE_FLAGS_DEFAULT =
static constexpr uint32_t LOAD_FLAGS_DEFAULT = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE); EPackedTextureFlags::TEXTURE_DEFAULT;
static constexpr uint32_t LOAD_FLAGS_DEFAULT =
EPackedLoadFlags::LOAD_VISIBLE | EPackedLoadFlags::LOAD_CACHE;
CUtlString m_EntryPath; CUtlString m_EntryPath;
uint16_t m_iPreloadSize; uint16_t m_iPreloadSize;
@ -56,17 +64,29 @@ struct VPKKeyValues_t
bool m_bUseCompression; bool m_bUseCompression;
bool m_bDeduplicate; bool m_bDeduplicate;
VPKKeyValues_t(const CUtlString& svEntryPath = "", uint16_t iPreloadSize = NULL, uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT, VPKKeyValues_t(const CUtlString& svEntryPath = "",
uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT, bool bUseCompression = true, bool bDeduplicate = true); uint16_t iPreloadSize = NULL,
uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT,
uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT,
bool bUseCompression = true, bool bDeduplicate = true);
}; };
//-----------------------------------------------------------------------------
// An asset packed into a VPK is carved into 'ENTRY_MAX_LEN' chunks, the chunk
// is then optionally compressed. A chunk is NOT compressed if the compressed
// size equals the uncompressed size.
//-----------------------------------------------------------------------------
struct VPKChunkDescriptor_t struct VPKChunkDescriptor_t
{ {
uint32_t m_nLoadFlags; // Load flags. uint32_t m_nLoadFlags;
uint16_t m_nTextureFlags; // Texture flags (only used if the entry is a vtf).
uint64_t m_nPackFileOffset; // Offset in pack file. // Texture flags (only used if the entry is a vtf).
uint64_t m_nCompressedSize; // Compressed size of chunk. uint16_t m_nTextureFlags;
uint64_t m_nUncompressedSize; // Uncompressed size of chunk.
// Offset in pack file.
uint64_t m_nPackFileOffset;
uint64_t m_nCompressedSize;
uint64_t m_nUncompressedSize;
VPKChunkDescriptor_t() VPKChunkDescriptor_t()
: m_nLoadFlags(0) : m_nLoadFlags(0)
@ -77,16 +97,27 @@ struct VPKChunkDescriptor_t
{ {
} }
VPKChunkDescriptor_t(FileHandle_t hDirectoryFile); VPKChunkDescriptor_t(FileHandle_t hDirectoryFile);
VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize); VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags,
uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize);
}; };
//-----------------------------------------------------------------------------
// An asset packed into a VPK is represented as an entry block.
//-----------------------------------------------------------------------------
struct VPKEntryBlock_t struct VPKEntryBlock_t
{ {
uint32_t m_nFileCRC; // Crc32 for the uncompressed entry. // Crc32 for the uncompressed entry.
uint16_t m_iPreloadSize; // Preload bytes. uint32_t m_nFileCRC;
uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry. uint16_t m_iPreloadSize;
CUtlVector<VPKChunkDescriptor_t> m_Fragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks).
CUtlString m_EntryPath; // Path to entry within vpk. // Index of the pack file that contains this entry.
uint16_t m_iPackFileIndex;
// Vector of all the chunks of a given entry
// (chunks have a size limit of 1 MiB, anything
// over this limit is fragmented into smaller chunks).
CUtlVector<VPKChunkDescriptor_t> m_Fragments;
CUtlString m_EntryPath;
VPKEntryBlock_t(FileHandle_t pFile, const char* svEntryPath); VPKEntryBlock_t(FileHandle_t pFile, const char* svEntryPath);
VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize, VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize,
@ -103,30 +134,30 @@ struct VPKEntryBlock_t
} }
}; };
//-----------------------------------------------------------------------------
// The VPK directory file header.
//-----------------------------------------------------------------------------
struct VPKDirHeader_t struct VPKDirHeader_t
{ {
uint32_t m_nHeaderMarker; // File magic. uint32_t m_nHeaderMarker; // File magic.
uint16_t m_nMajorVersion; // Vpk major version. uint16_t m_nMajorVersion; // Vpk major version.
uint16_t m_nMinorVersion; // Vpk minor version. uint16_t m_nMinorVersion; // Vpk minor version.
uint32_t m_nDirectorySize; // Directory tree size. uint32_t m_nDirectorySize; // Directory tree size.
uint32_t m_nSignatureSize; // Directory signature. uint32_t m_nSignatureSize; // Directory signature.
};
struct VPKPair_t
{
CUtlString m_PackName;
CUtlString m_DirName;
VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch);
}; };
//-----------------------------------------------------------------------------
// The VPK directory tree structure.
//-----------------------------------------------------------------------------
struct VPKDir_t struct VPKDir_t
{ {
VPKDirHeader_t m_Header; // Dir header. VPKDirHeader_t m_Header;
CUtlVector<VPKEntryBlock_t> m_EntryBlocks; // Vector of entry blocks. CUtlVector<VPKEntryBlock_t> m_EntryBlocks;
uint16_t m_PackFileCount; // Number of pack patches (pack file count-1). CUtlString m_DirFilePath;
CUtlVector<CUtlString> m_PackFiles; // Vector of pack file names.
CUtlString m_DirFilePath; // Path to vpk_dir file. // This set only contains packfile indices used
// by the directory tree, notated as pak000_xxx.
std::set<uint16_t> m_PakFileIndices;
class CTreeBuilder class CTreeBuilder
{ {
@ -145,7 +176,6 @@ struct VPKDir_t
{ {
m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION; m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION;
m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL; m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL;
m_PackFileCount = NULL;
}; };
VPKDir_t(const CUtlString& svDirectoryFile); VPKDir_t(const CUtlString& svDirectoryFile);
VPKDir_t(const CUtlString& svDirectoryFile, bool bSanitizeName); VPKDir_t(const CUtlString& svDirectoryFile, bool bSanitizeName);
@ -153,7 +183,7 @@ struct VPKDir_t
void Init(const CUtlString& svPath); void Init(const CUtlString& svPath);
CUtlString StripLocalePrefix(const CUtlString& svDirectoryFile) const; CUtlString StripLocalePrefix(const CUtlString& svDirectoryFile) const;
CUtlString GetPackFile(const CUtlString& svDirectoryPath, uint16_t iPackFileIndex) const; CUtlString GetPackFileNameForIndex(uint16_t iPackFileIndex) const;
void WriteHeader(FileHandle_t hDirectoryFile) const; void WriteHeader(FileHandle_t hDirectoryFile) const;
void WriteTreeSize(FileHandle_t hDirectoryFile) const; void WriteTreeSize(FileHandle_t hDirectoryFile) const;
@ -161,6 +191,22 @@ struct VPKDir_t
void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector<VPKEntryBlock_t>& entryBlocks); void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector<VPKEntryBlock_t>& entryBlocks);
}; };
//-----------------------------------------------------------------------------
// Contains the VPK directory name, and the pack file name. Used for building
// the VPK file.
// !TODO[ AMOS ]: Remove this when patching is implemented!
//-----------------------------------------------------------------------------
struct VPKPair_t
{
CUtlString m_PackName;
CUtlString m_DirName;
VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch);
};
//-----------------------------------------------------------------------------
// VPK utility class.
//-----------------------------------------------------------------------------
class CPackedStore class CPackedStore
{ {
public: public:
@ -190,11 +236,11 @@ public:
void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = ""); void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = "");
private: private:
lzham_compress_params m_lzCompParams; // LZham compression parameters. lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters. lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap; std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap;
}; };
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
extern CPackedStore* g_pPackedStore; extern CPackedStore* g_pPackedStore;
#endif // PACKEDSTORE_H #endif // PACKEDSTORE_H