R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

VPK utility code improvements

Browse Source 
- Only process referenced pack files, previously, if an entry in the VPK file was build into a pack of index 128, the code would generate a list of pack files from 0 to 128.
- Added documentation to VPK structure.
This commit is contained in:
Kawe Mazidjatari 2023-07-06 00:49:17 +02:00
parent 5939e05331
commit c215fcc171
3 changed files with 99 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
r5dev/public/ipackedstore.h
View File

@ -1,7 +1,7 @@
#ifndef IPACKEDSTORE_H
#define IPACKEDSTORE_H
enum class EPackedLoadFlags : int
enum EPackedLoadFlags
{
LOAD_NONE,
LOAD_VISIBLE = 1 << 0, // Visible to FileSystem.
@ -11,7 +11,7 @@ enum class EPackedLoadFlags : int
LOAD_TEXTURE_UNK2 = 1 << 20,
};
enum class EPackedTextureFlags : short
enum EPackedTextureFlags
{
TEXTURE_NONE,
TEXTURE_DEFAULT = 1 << 3,

36
r5dev/vpklib/packedstore.cpp
View File

@ -582,9 +582,9 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
BuildManifest(vpkDir.m_EntryBlocks, workspacePath, GetLevelName(vpkDir.m_DirFilePath));
const CUtlString basePath = vpkDir.m_DirFilePath.StripFilename(false);
FOR_EACH_VEC(vpkDir.m_PackFiles, i)
for (uint16_t packFileIndex : vpkDir.m_PakFileIndices)
{
const CUtlString packFile = basePath + vpkDir.m_PackFiles[i];
const CUtlString packFile = basePath + vpkDir.GetPackFileNameForIndex(packFileIndex);
// Read from each pack file.
FileHandle_t hPackFile = FileSystem()->Open(packFile.Get(), "rb", "GAME");
@ -598,7 +598,7 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
{
const VPKEntryBlock_t& entryBlock = vpkDir.m_EntryBlocks[j];
if (entryBlock.m_iPackFileIndex != uint16_t(i))
if (entryBlock.m_iPackFileIndex != packFileIndex)
{
// Chunk doesn't belongs to this block.
continue;
@ -616,7 +616,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
continue;
}
DevMsg(eDLL_T::FS, "Unpacking entry '%i' from block '%i' ('%s')\n", j, i, entryBlock.m_EntryPath.Get());
DevMsg(eDLL_T::FS, "Unpacking entry '%i' from block '%i' ('%s')\n",
j, entryBlock.m_iPackFileIndex, entryBlock.m_EntryPath.Get());
FOR_EACH_VEC(entryBlock.m_Fragments, k)
{
@ -642,8 +643,8 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspace
if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
{
Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk not decompressed)\n",
lzDecompStatus, k, j, i);
Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%i' within entry '%i' in block '%hu' (chunk not decompressed)\n",
lzDecompStatus, k, j, packFileIndex);
}
else // If successfully decompressed, write to file.
{
@ -906,37 +907,26 @@ void VPKDir_t::Init(const CUtlString& dirFilePath)
FileSystem()->Read(&m_Header.m_nSignatureSize, sizeof(uint32_t), hDirFile); //
g_pPackedStore->GetEntryBlocks(m_EntryBlocks, hDirFile);
m_DirFilePath = dirFilePath; // Set path to vpk directory file.
m_PackFileCount = 0;
// Obtain every referenced pack file from the directory tree.
FOR_EACH_VEC(m_EntryBlocks, i)
{
const VPKEntryBlock_t& entryBlock = m_EntryBlocks[i];
if (entryBlock.m_iPackFileIndex > m_PackFileCount)
{
m_PackFileCount = entryBlock.m_iPackFileIndex;
}
}
for (uint16_t i = 0; i < m_PackFileCount + 1; i++)
{
m_PackFiles.AddToTail(GetPackFile(dirFilePath, i));
m_PakFileIndices.insert(entryBlock.m_iPackFileIndex);
}
FileSystem()->Close(hDirFile);
}
//-----------------------------------------------------------------------------
// Purpose: formats pack file path for specific directory file
// Input : &directoryPath -
// iPackFileIndex -
// Purpose: formats pack file path for specified patch
// Input : iPackFileIndex - (patch)
// output : string
//-----------------------------------------------------------------------------
CUtlString VPKDir_t::GetPackFile(const CUtlString& directoryPath, uint16_t iPackFileIndex) const
CUtlString VPKDir_t::GetPackFileNameForIndex(uint16_t iPackFileIndex) const
{
CUtlString packChunkName = StripLocalePrefix(directoryPath);
CUtlString packChunkName = StripLocalePrefix(m_DirFilePath);
CUtlString packChunkIndex;
packChunkIndex.Format("pak000_%03d", iPackFileIndex);

122
r5dev/vpklib/packedstore.h
View File

@ -44,10 +44,18 @@ static const char* const DIR_LOCALE[]
"tchinese"
};
//-----------------------------------------------------------------------------
// KeyValues structure for the VPK manifest file. This struct gets populated by
// the VPK's corresponding manifest file, which ultimately determines how each
// asset is getting packed into the VPK.
//-----------------------------------------------------------------------------
struct VPKKeyValues_t
{
static constexpr uint16_t TEXTURE_FLAGS_DEFAULT = static_cast<uint16_t>(EPackedTextureFlags::TEXTURE_DEFAULT);
static constexpr uint32_t LOAD_FLAGS_DEFAULT = static_cast<uint32_t>(EPackedLoadFlags::LOAD_VISIBLE) | static_cast<uint32_t>(EPackedLoadFlags::LOAD_CACHE);
static constexpr uint16_t TEXTURE_FLAGS_DEFAULT =
EPackedTextureFlags::TEXTURE_DEFAULT;
static constexpr uint32_t LOAD_FLAGS_DEFAULT =
EPackedLoadFlags::LOAD_VISIBLE | EPackedLoadFlags::LOAD_CACHE;
CUtlString m_EntryPath;
uint16_t m_iPreloadSize;
@ -56,17 +64,29 @@ struct VPKKeyValues_t
bool m_bUseCompression;
bool m_bDeduplicate;
VPKKeyValues_t(const CUtlString& svEntryPath = "", uint16_t iPreloadSize = NULL, uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT,
uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT, bool bUseCompression = true, bool bDeduplicate = true);
VPKKeyValues_t(const CUtlString& svEntryPath = "",
uint16_t iPreloadSize = NULL,
uint32_t nLoadFlags = LOAD_FLAGS_DEFAULT,
uint16_t nTextureFlags = TEXTURE_FLAGS_DEFAULT,
bool bUseCompression = true, bool bDeduplicate = true);
};
//-----------------------------------------------------------------------------
// An asset packed into a VPK is carved into 'ENTRY_MAX_LEN' chunks, the chunk
// is then optionally compressed. A chunk is NOT compressed if the compressed
// size equals the uncompressed size.
//-----------------------------------------------------------------------------
struct VPKChunkDescriptor_t
{
uint32_t m_nLoadFlags; // Load flags.
uint16_t m_nTextureFlags; // Texture flags (only used if the entry is a vtf).
uint64_t m_nPackFileOffset; // Offset in pack file.
uint64_t m_nCompressedSize; // Compressed size of chunk.
uint64_t m_nUncompressedSize; // Uncompressed size of chunk.
uint32_t m_nLoadFlags;
// Texture flags (only used if the entry is a vtf).
uint16_t m_nTextureFlags;
// Offset in pack file.
uint64_t m_nPackFileOffset;
uint64_t m_nCompressedSize;
uint64_t m_nUncompressedSize;
VPKChunkDescriptor_t()
: m_nLoadFlags(0)
@ -77,16 +97,27 @@ struct VPKChunkDescriptor_t
{
}
VPKChunkDescriptor_t(FileHandle_t hDirectoryFile);
VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags, uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize);
VPKChunkDescriptor_t(uint32_t nLoadFlags, uint16_t nTextureFlags,
uint64_t nPackFileOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize);
};
//-----------------------------------------------------------------------------
// An asset packed into a VPK is represented as an entry block.
//-----------------------------------------------------------------------------
struct VPKEntryBlock_t
{
uint32_t m_nFileCRC; // Crc32 for the uncompressed entry.
uint16_t m_iPreloadSize; // Preload bytes.
uint16_t m_iPackFileIndex; // Index of the pack file that contains this entry.
CUtlVector<VPKChunkDescriptor_t> m_Fragments; // Vector of all the chunks of a given entry (chunks have a size limit of 1 MiB, anything over this limit is fragmented into smaller chunks).
CUtlString m_EntryPath; // Path to entry within vpk.
// Crc32 for the uncompressed entry.
uint32_t m_nFileCRC;
uint16_t m_iPreloadSize;
// Index of the pack file that contains this entry.
uint16_t m_iPackFileIndex;
// Vector of all the chunks of a given entry
// (chunks have a size limit of 1 MiB, anything
// over this limit is fragmented into smaller chunks).
CUtlVector<VPKChunkDescriptor_t> m_Fragments;
CUtlString m_EntryPath;
VPKEntryBlock_t(FileHandle_t pFile, const char* svEntryPath);
VPKEntryBlock_t(const uint8_t* pData, size_t nLen, int64_t nOffset, uint16_t iPreloadSize,
@ -103,30 +134,30 @@ struct VPKEntryBlock_t
}
};
//-----------------------------------------------------------------------------
// The VPK directory file header.
//-----------------------------------------------------------------------------
struct VPKDirHeader_t
{
uint32_t m_nHeaderMarker; // File magic.
uint16_t m_nMajorVersion; // Vpk major version.
uint16_t m_nMinorVersion; // Vpk minor version.
uint32_t m_nDirectorySize; // Directory tree size.
uint32_t m_nSignatureSize; // Directory signature.
};
struct VPKPair_t
{
CUtlString m_PackName;
CUtlString m_DirName;
VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch);
uint32_t m_nHeaderMarker; // File magic.
uint16_t m_nMajorVersion; // Vpk major version.
uint16_t m_nMinorVersion; // Vpk minor version.
uint32_t m_nDirectorySize; // Directory tree size.
uint32_t m_nSignatureSize; // Directory signature.
};
//-----------------------------------------------------------------------------
// The VPK directory tree structure.
//-----------------------------------------------------------------------------
struct VPKDir_t
{
VPKDirHeader_t m_Header; // Dir header.
CUtlVector<VPKEntryBlock_t> m_EntryBlocks; // Vector of entry blocks.
uint16_t m_PackFileCount; // Number of pack patches (pack file count-1).
CUtlVector<CUtlString> m_PackFiles; // Vector of pack file names.
CUtlString m_DirFilePath; // Path to vpk_dir file.
VPKDirHeader_t m_Header;
CUtlVector<VPKEntryBlock_t> m_EntryBlocks;
CUtlString m_DirFilePath;
// This set only contains packfile indices used
// by the directory tree, notated as pak000_xxx.
std::set<uint16_t> m_PakFileIndices;
class CTreeBuilder
{
@ -145,7 +176,6 @@ struct VPKDir_t
{
m_Header.m_nHeaderMarker = VPK_HEADER_MARKER; m_Header.m_nMajorVersion = VPK_MAJOR_VERSION;
m_Header.m_nMinorVersion = VPK_MINOR_VERSION; m_Header.m_nDirectorySize = NULL, m_Header.m_nSignatureSize = NULL;
m_PackFileCount = NULL;
};
VPKDir_t(const CUtlString& svDirectoryFile);
VPKDir_t(const CUtlString& svDirectoryFile, bool bSanitizeName);
@ -153,7 +183,7 @@ struct VPKDir_t
void Init(const CUtlString& svPath);
CUtlString StripLocalePrefix(const CUtlString& svDirectoryFile) const;
CUtlString GetPackFile(const CUtlString& svDirectoryPath, uint16_t iPackFileIndex) const;
CUtlString GetPackFileNameForIndex(uint16_t iPackFileIndex) const;
void WriteHeader(FileHandle_t hDirectoryFile) const;
void WriteTreeSize(FileHandle_t hDirectoryFile) const;
@ -161,6 +191,22 @@ struct VPKDir_t
void BuildDirectoryFile(const CUtlString& svDirectoryFile, const CUtlVector<VPKEntryBlock_t>& entryBlocks);
};
//-----------------------------------------------------------------------------
// Contains the VPK directory name, and the pack file name. Used for building
// the VPK file.
// !TODO[ AMOS ]: Remove this when patching is implemented!
//-----------------------------------------------------------------------------
struct VPKPair_t
{
CUtlString m_PackName;
CUtlString m_DirName;
VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch);
};
//-----------------------------------------------------------------------------
// VPK utility class.
//-----------------------------------------------------------------------------
class CPackedStore
{
public:
@ -190,11 +236,11 @@ public:
void UnpackWorkspace(const VPKDir_t& vpkDir, const char* workspaceName = "");
private:
lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap;
};
///////////////////////////////////////////////////////////////////////////////
extern CPackedStore* g_pPackedStore;
#endif // PACKEDSTORE_H
#endif // PACKEDSTORE_H