diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp
index 4c73ac25..bae5acda 100644
--- a/r5dev/vpklib/packedstore.cpp
+++ b/r5dev/vpklib/packedstore.cpp
@@ -345,6 +345,31 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetPath, const uint
}
}
+//-----------------------------------------------------------------------------
+// Purpose: attempts to deduplicate a chunk of data by comparing it to existing chunks
+// Input : *pEntryBuffer -
+// &descriptor -
+// chunkIndex
+// Output : true if the chunk was deduplicated, false otherwise
+//-----------------------------------------------------------------------------
+bool CPackedStore::Deduplicate(const uint8_t* pEntryBuffer, VPKChunkDescriptor_t& descriptor, const size_t chunkIndex)
+{
+ string entryHash(reinterpret_cast<const char*>(pEntryBuffer), descriptor.m_nUncompressedSize);
+ entryHash = sha1(entryHash);
+
+ auto p = m_mChunkHashMap.insert({ entryHash, descriptor });
+ if (!p.second) // Map to existing chunk to avoid having copies of the same data.
+ {
+ DevMsg(eDLL_T::FS, "Mapping chunk '%zu' ('%s') to existing chunk at '0x%llx'\n",
+ chunkIndex, entryHash.c_str(), p.first->second.m_nPackFileOffset);
+ descriptor = p.first->second;
+
+ return true;
+ }
+
+ return false;
+}
+
//-----------------------------------------------------------------------------
// Purpose: packs all files from workspace path into VPK file
// Input : &vPair -
@@ -354,7 +379,7 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetPath, const uint
//-----------------------------------------------------------------------------
void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly)
{
- const string svPackFilePath = string(svBuildPath + vPair.m_svPackName);
+ const string svPackFilePath(svBuildPath + vPair.m_svPackName);
FileHandle_t hPackFile = FileSystem()->Open(svPackFilePath.c_str(), "wb", "GAME");
if (!hPackFile)
@@ -367,6 +392,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
if (!pEntryBuffer)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to allocate memory for entry buffer!\n", __FUNCTION__);
+ FileSystem()->Close(hPackFile);
return;
}
@@ -420,21 +446,13 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
FileSystem()->Read(pEntryBuffer, int(vDescriptor.m_nCompressedSize), hAsset);
vDescriptor.m_nPackFileOffset = FileSystem()->Tell(hPackFile);
- if (vEntryValue.m_bDeduplicate)
+ if (vEntryValue.m_bDeduplicate && Deduplicate(pEntryBuffer, vDescriptor, j))
{
- string svEntryHash = sha1(string(reinterpret_cast<char*>(pEntryBuffer), vDescriptor.m_nUncompressedSize));
- auto p = m_mChunkHashMap.insert({ svEntryHash, vDescriptor });
+ nSharedTotal += vDescriptor.m_nCompressedSize;
+ nSharedCount++;
- if (!p.second) // Map to existing chunk to avoid having copies of the same data.
- {
- DevMsg(eDLL_T::FS, "Mapping chunk '%zu' ('%s') to existing chunk at '0x%llx'\n", j, svEntryHash.c_str(), p.first->second.m_nPackFileOffset);
-
- vDescriptor = p.first->second;
- nSharedTotal += vDescriptor.m_nCompressedSize;
- nSharedCount++;
-
- continue;
- }
+ // Data was deduplicated.
+ continue;
}
if (vEntryValue.m_bUseCompression)
@@ -515,63 +533,61 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDirectory, const string& svW
const VPKEntryBlock_t& vEntryBlock = vDirectory.m_vEntryBlocks[j];
if (vEntryBlock.m_iPackFileIndex != static_cast<uint16_t>(i))
{
+ // Chunk doesn't belongs to this block.
continue;
}
- else // Chunk belongs to this block.
+
+ string svFilePath;
+ CreateDirectories(svWorkspace + vEntryBlock.m_svEntryPath, &svFilePath);
+ FileHandle_t hAsset = FileSystem()->Open(svFilePath.c_str(), "wb", "GAME");
+
+ if (!hAsset)
{
- string svFilePath;
- CreateDirectories(svWorkspace + vEntryBlock.m_svEntryPath, &svFilePath);
- FileHandle_t hAsset = FileSystem()->Open(svFilePath.c_str(), "wb", "GAME");
+ Error(eDLL_T::FS, NO_ERROR, "%s - Unable to write to '%s' (read-only?)\n", __FUNCTION__, svFilePath.c_str());
+ continue;
+ }
- if (!hAsset)
+ DevMsg(eDLL_T::FS, "Unpacking entry '%zu' from block '%zu' ('%s')\n", j, i, vEntryBlock.m_svEntryPath.c_str());
+ for (size_t k = 0, cs = vEntryBlock.m_vFragments.size(); k < cs; k++)
+ {
+ const VPKChunkDescriptor_t& vChunk = vEntryBlock.m_vFragments[k];
+ m_nChunkCount++;
+
+ FileSystem()->Seek(hPackFile, int(vChunk.m_nPackFileOffset), FileSystemSeek_t::FILESYSTEM_SEEK_HEAD);
+ FileSystem()->Read(pSourceBuffer, int(vChunk.m_nCompressedSize), hPackFile);
+
+ if (vChunk.m_nCompressedSize == vChunk.m_nUncompressedSize) // Data is not compressed.
{
- Error(eDLL_T::FS, NO_ERROR, "%s - Unable to write to '%s' (read-only?)\n", __FUNCTION__, svFilePath.c_str());
- continue;
+ FileSystem()->Write(pSourceBuffer, int(vChunk.m_nUncompressedSize), hAsset);
+ break;
}
- DevMsg(eDLL_T::FS, "Unpacking entry '%zu' from block '%zu' ('%s')\n", j, i, vEntryBlock.m_svEntryPath.c_str());
- for (size_t k = 0, cs = vEntryBlock.m_vFragments.size(); k < cs; k++)
+ size_t nDstLen = ENTRY_MAX_LEN;
+ assert(vChunk.m_nCompressedSize <= nDstLen);
+
+ if (vChunk.m_nCompressedSize > nDstLen)
+ break; // Corrupt or invalid chunk descriptor.
+
+ lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, pDestBuffer,
+ &nDstLen, pSourceBuffer, vChunk.m_nCompressedSize, nullptr);
+
+ if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
{
- const VPKChunkDescriptor_t& vChunk = vEntryBlock.m_vFragments[k];
- m_nChunkCount++;
-
- FileSystem()->Seek(hPackFile, int(vChunk.m_nPackFileOffset), FileSystemSeek_t::FILESYSTEM_SEEK_HEAD);
- FileSystem()->Read(pSourceBuffer, int(vChunk.m_nCompressedSize), hPackFile);
-
- if (vChunk.m_nCompressedSize != vChunk.m_nUncompressedSize) // Data is compressed.
- {
- size_t nDstLen = ENTRY_MAX_LEN;
- assert(vChunk.m_nCompressedSize <= nDstLen);
-
- if (vChunk.m_nCompressedSize > nDstLen)
- break; // Corrupt or invalid chunk descriptor.
-
- lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, pDestBuffer,
- &nDstLen, pSourceBuffer, vChunk.m_nCompressedSize, nullptr);
-
- if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
- {
- Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk not decompressed)\n",
- lzDecompStatus, m_nChunkCount, i, vEntryBlock.m_iPackFileIndex);
- }
- else // If successfully decompressed, write to file.
- {
- FileSystem()->Write(pDestBuffer, int(nDstLen), hAsset);
- }
- }
- else // If not compressed, write source data into output file.
- {
- FileSystem()->Write(pSourceBuffer, int(vChunk.m_nUncompressedSize), hAsset);
- }
+ Error(eDLL_T::FS, NO_ERROR, "Status '%d' for chunk '%zu' within entry '%zu' in block '%hu' (chunk not decompressed)\n",
+ lzDecompStatus, m_nChunkCount, i, vEntryBlock.m_iPackFileIndex);
}
-
- FileSystem()->Close(hAsset);
- if (m_nChunkCount == vEntryBlock.m_vFragments.size()) // Only validate after last entry in block had been written.
+ else // If successfully decompressed, write to file.
{
- m_nChunkCount = NULL;
- ValidateCRC32PostDecomp(svFilePath, vEntryBlock.m_nFileCRC);
+ FileSystem()->Write(pDestBuffer, int(nDstLen), hAsset);
}
}
+
+ FileSystem()->Close(hAsset);
+ if (m_nChunkCount == vEntryBlock.m_vFragments.size()) // Only validate after last entry in block had been written.
+ {
+ m_nChunkCount = NULL;
+ ValidateCRC32PostDecomp(svFilePath, vEntryBlock.m_nFileCRC);
+ }
}
FileSystem()->Close(hPackFile);
}
diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h
index 1d4746b8..63d3607e 100644
--- a/r5dev/vpklib/packedstore.h
+++ b/r5dev/vpklib/packedstore.h
@@ -150,18 +150,19 @@ public:
vector<string> GetIgnoreList(const string& svWorkspace) const;
string FormatEntryPath(const string& svPath, const string& svName, const string& svExtension) const;
-
void BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const string& svWorkspace, const string& svManifestName) const;
+ void ValidateCRC32PostDecomp(const string& svAssetPath, const uint32_t nFileCRC);
+ bool Deduplicate(const uint8_t* pEntryBuffer, VPKChunkDescriptor_t& descriptor, const size_t chunkIndex);
+
void PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly);
void UnpackWorkspace(const VPKDir_t& vDirectory, const string& svWorkspace = "");
- void ValidateCRC32PostDecomp(const string& svAssetPath, const uint32_t nFileCRC);
private:
size_t m_nChunkCount; // The number of fragments for this asset.
lzham_compress_params m_lzCompParams; // LZham compression parameters.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.
- std::unordered_map<string, VPKChunkDescriptor_t&> m_mChunkHashMap;
+ std::unordered_map<string, const VPKChunkDescriptor_t&> m_mChunkHashMap;
};
///////////////////////////////////////////////////////////////////////////////
extern CPackedStore* g_pPackedStore;