From 14f409c8c6c7d772174ef70ef218a092434d905d Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Thu, 2 Jun 2022 02:00:35 +0200 Subject: [PATCH] Update packedstore.cpp --- r5dev/vpklib/packedstore.cpp | 338 +++++++++++++++++++++++++++++++++++ 1 file changed, 338 insertions(+) diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index 74438d3f..557e8b01 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -189,5 +189,343 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile) } } +void CPackedStore::PackAll(string svDirIn, string svPathOut) +{ + CIOStream writer("client_mp_rr_canyonlands_staging.bsp.pak000_000.vpk", CIOStream::Mode_t::WRITE); + + vector vEntryBlocks; + vector vPaths = GetEntryPaths(svDirIn); + + for (size_t i = 0; i < vPaths.size(); i++) + { + CIOStream reader(vPaths[i], CIOStream::Mode_t::READ); + if (reader.IsReadable()) + { + vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), 0, 0x101, 0, StringReplaceC(vPaths[i], fs_packedstore_workspace->GetString(), ""))); + for (size_t j = 0; j < vEntryBlocks[i].m_vvEntries.size(); j++) + { + uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; + uint8_t* pDest = new uint8_t[DECOMP_MAX_BUF]; + + reader.Read(*pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); + vEntryBlocks[i].m_vvEntries[j].m_nArchiveOffset = writer.GetPosition(); + + m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize, pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); + if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) + { + Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), vEntryBlocks.at(i).m_iArchiveIndex); + Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d' (file will be packed without compression.)\n", m_lzCompStatus); + + vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize = vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; + writer.Write(pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); + } + else + writer.Write(pDest, vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize); + + vEntryBlocks[i].m_vvEntries[j].m_bIsCompressed = vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize != vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; + + delete[] pSrc; + delete[] pDest; + } + } + } + + VPKDir_t vDir = VPKDir_t(); + vDir.Build("englishclient_mp_rr_canyonlands_staging.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <> !!!] +} + +//----------------------------------------------------------------------------- +// Purpose: extracts all files from specified vpk file +//----------------------------------------------------------------------------- +void CPackedStore::UnpackAll(VPKDir_t vpkDir, string svPathOut) +{ + for (int i = 0; i < vpkDir.m_vsvArchives.size(); i++) + { + fs::path fspVpkPath(vpkDir.m_svDirPath); + string svPath = fspVpkPath.parent_path().u8string() + "\\" + vpkDir.m_vsvArchives[i]; + ifstream packChunkStream(svPath, std::ios_base::binary); // Create stream to read from each archive. + + for ( VPKEntryBlock_t block : vpkDir.m_vvEntryBlocks) + { + // Escape if block archive index is not part of the extracting archive chunk index. + if (block.m_iArchiveIndex != i) { goto escape; } + else + { + string svFilePath = CreateDirectories(svPathOut + "\\" + block.m_svBlockPath); + ofstream outFileStream(svFilePath, std::ios_base::binary | std::ios_base::out); + + if (!outFileStream.is_open()) + { + Error(eDLL_T::FS, "Error: unable to access file '%s'!\n", svFilePath.c_str()); + } + outFileStream.clear(); // Make sure file is empty before writing. + for (VPKEntryDescriptor_t entry : block.m_vvEntries) + { + char* pCompressedData = new char[entry.m_nCompressedSize]; + memset(pCompressedData, 0, entry.m_nCompressedSize); // Compressed region. + + packChunkStream.seekg(entry.m_nArchiveOffset); // Seek to entry offset in archive. + packChunkStream.read(pCompressedData, entry.m_nCompressedSize); // Read compressed data from archive. + + if (entry.m_bIsCompressed) + { + lzham_uint8* pLzOutputBuf = new lzham_uint8[entry.m_nUncompressedSize]; + m_lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, pLzOutputBuf, + (size_t*)&entry.m_nUncompressedSize, (lzham_uint8*)pCompressedData, + entry.m_nCompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); + + if (fs_packedstore_entryblock_stats->GetBool()) + { + DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); + DevMsg(eDLL_T::FS, "] Block path : '%s'\n", block.m_svBlockPath.c_str()); + DevMsg(eDLL_T::FS, "] Entry count : '%llu'\n", block.m_vvEntries.size()); + DevMsg(eDLL_T::FS, "] Compressed size : '%llu'\n", entry.m_nCompressedSize); + DevMsg(eDLL_T::FS, "] Uncompressed size : '%llu'\n", entry.m_nUncompressedSize); + DevMsg(eDLL_T::FS, "] Static CRC32 hash : '0x%lX'\n", block.m_nCrc32); + DevMsg(eDLL_T::FS, "] Computed CRC32 hash : '0x%lX'\n", m_nCrc32_Internal); + DevMsg(eDLL_T::FS, "] Computed ADLER32 hash : '0x%lX'\n", m_nAdler32_Internal); + DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); + } + + if (block.m_vvEntries.size() == 1) // Internal checksum can only match block checksum if entry size is 1. + { + if (block.m_nCrc32 != m_nCrc32_Internal) + { + Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", block.m_svBlockPath.c_str(), m_nCrc32_Internal, block.m_nCrc32); + } + } + else { m_nEntryCount++; } + + if (m_lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS) + { + Error(eDLL_T::FS, "Error: failed decompression for an entry within block '%s' in archive '%d'!\n", block.m_svBlockPath.c_str(), i); + Error(eDLL_T::FS, "'lzham::lzham_lib_decompress_memory' returned with status '%d'.\n", m_lzDecompStatus); + } + else + { + // If successfully decompressed, write to file. + outFileStream.write((char*)pLzOutputBuf, entry.m_nUncompressedSize); + } + delete[] pLzOutputBuf; + } + else + { + // If not compressed, write raw data into output file. + outFileStream.write(pCompressedData, entry.m_nUncompressedSize); + } + delete[] pCompressedData; + } + outFileStream.close(); + if (m_nEntryCount == block.m_vvEntries.size()) // Only validate after last entry in block had been written. + { + // Set internal hash to precomputed entry hash for post decompress validation. + m_nCrc32_Internal = block.m_nCrc32; + + ValidateCRC32PostDecomp(svFilePath); + //ValidateAdler32PostDecomp(svFilePath); + m_nEntryCount = 0; + } + }escape:; + } + packChunkStream.close(); + } +} + +//----------------------------------------------------------------------------- +// Purpose: 'vpk_entry_block' constructor +//----------------------------------------------------------------------------- +VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* reader, string svPath) +{ + std::replace(svPath.begin(), svPath.end(), '/', '\\'); // Flip forward slashes in filepath to windows-style backslash. + + this->m_svBlockPath = svPath; // Set path of block. + reader->Read(this->m_nCrc32); // + reader->Read(this->m_nPreloadBytes); // + reader->Read(this->m_iArchiveIndex); // + + do // Loop through all entries in the block and push them to the vector. + { + VPKEntryDescriptor_t entry(reader); + this->m_vvEntries.push_back(entry); + } while (reader->Read() != 0xFFFF); +} +#undef min +VPKEntryBlock_t::VPKEntryBlock_t(const vector &vData, int64_t nOffset, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, string svBlockPath) +{ + m_nCrc32 = crc32::update(m_nCrc32, vData.data(), vData.size()); + m_nPreloadBytes = 0; + m_iArchiveIndex = nArchiveIndex; + m_svBlockPath = svBlockPath; + + int nEntryCount = (vData.size() + RVPK_MAX_BLOCK - 1) / RVPK_MAX_BLOCK; + uint64_t nDataSize = vData.size(); + int64_t nCurrentOffset = nOffset; + for (int i = 0; i < nEntryCount; i++) + { + uint64_t nSize = std::min(RVPK_MAX_BLOCK, nDataSize); + nDataSize -= nSize; + m_vvEntries.push_back(VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize)); + nCurrentOffset += nSize; + } +} + +VPKEntryDescriptor_t::VPKEntryDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nArchiveOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize) +{ + m_nEntryFlags = nEntryFlags; + m_nTextureFlags = nTextureFlags; + m_nArchiveOffset = nArchiveOffset; + + m_nCompressedSize = nCompressedSize; + m_nUncompressedSize = nUncompressedSize; +} + +//----------------------------------------------------------------------------- +// Purpose: 'VPKDir_t' constructor +// Input : *pReader - +//----------------------------------------------------------------------------- +VPKEntryDescriptor_t::VPKEntryDescriptor_t(CIOStream* pReader) +{ + pReader->Read(this->m_nEntryFlags); // + pReader->Read(this->m_nTextureFlags); // + pReader->Read(this->m_nArchiveOffset); // + pReader->Read(this->m_nCompressedSize); // + pReader->Read(this->m_nUncompressedSize); // + this->m_bIsCompressed = (this->m_nCompressedSize != this->m_nUncompressedSize); +} + +//----------------------------------------------------------------------------- +// Purpose: 'VPKDir_t' file constructor +//----------------------------------------------------------------------------- +VPKDir_t::VPKDir_t(const string& svPath) +{ + CIOStream reader(svPath, CIOStream::Mode_t::READ); + reader.Read(this->m_vHeader.m_nFileMagic); + + if (this->m_vHeader.m_nFileMagic != RVPK_DIR_MAGIC) + { + Error(eDLL_T::FS, "Error: vpk_dir file '%s' has invalid magic!\n", svPath.c_str()); + //return; + } + + reader.Read(this->m_vHeader.m_nMajorVersion); // + reader.Read(this->m_vHeader.m_nMinorVersion); // + reader.Read(this->m_vHeader.m_nTreeSize); // + reader.Read(this->m_nFileDataSize); // + + DevMsg(eDLL_T::FS, "______________________________________________________________\n"); + DevMsg(eDLL_T::FS, "] HEADER_DETAILS ---------------------------------------------\n"); + DevMsg(eDLL_T::FS, "] File Magic : '%lu'\n", this->m_vHeader.m_nFileMagic); + DevMsg(eDLL_T::FS, "] Major Version : '%hu'\n", this->m_vHeader.m_nMajorVersion); + DevMsg(eDLL_T::FS, "] Minor Version : '%hu'\n", this->m_vHeader.m_nMinorVersion); + DevMsg(eDLL_T::FS, "] Tree Size : '%lu'\n", this->m_vHeader.m_nTreeSize); + DevMsg(eDLL_T::FS, "] File Data Size : '%lu'\n", this->m_nFileDataSize); + + this->m_vvEntryBlocks = g_pPackedStore->GetEntryBlocks(&reader); + this->m_svDirPath = svPath; // Set path to vpk_dir file. + + for (VPKEntryBlock_t block : this->m_vvEntryBlocks) + { + if (block.m_iArchiveIndex > this->m_iArchiveCount) + { + this->m_iArchiveCount = block.m_iArchiveIndex; + } + } + + DevMsg(eDLL_T::FS, "______________________________________________________________\n"); + DevMsg(eDLL_T::FS, "] PACK_CHUNKS ------------------------------------------------\n"); + + for (int i = 0; i < this->m_iArchiveCount + 1; i++) + { + string svArchivePath = g_pPackedStore->GetPackChunkFile(svPath, i); + DevMsg(eDLL_T::FS, "] '%s'\n", svArchivePath.c_str()); + this->m_vsvArchives.push_back(svArchivePath); + } +} + +//----------------------------------------------------------------------------- +// Purpose: builds the VPKDir file +// Input : &svFileName - +// &vEntryBlocks - +//----------------------------------------------------------------------------- +void VPKDir_t::Build(const string& svFileName, const vector& vEntryBlocks) +{ + CIOStream writer(svFileName, CIOStream::Mode_t::WRITE); + auto vMap = std::map>>(); + + writer.Write(this->m_vHeader.m_nFileMagic); + writer.Write(this->m_vHeader.m_nMajorVersion); + writer.Write(this->m_vHeader.m_nMinorVersion); + writer.Write(this->m_vHeader.m_nTreeSize); + writer.Write(this->m_vHeader.m_nTreeSize); + + for (VPKEntryBlock_t vBlock : vEntryBlocks) + { + string svExtension = GetExtension(vBlock.m_svBlockPath); + string svFileName = GetFileName(vBlock.m_svBlockPath, true); + string svFilePath = RemoveFileName(vBlock.m_svBlockPath); + + if (svFilePath.empty()) + { + svFilePath = " "; // Has to be padded with a space character if empty. + } + if (!vMap.count(svExtension)) + { + vMap.insert({ svExtension, std::map>() }); + } + if (!vMap[svExtension].count(svFilePath)) + { + vMap[svExtension].insert({ svFilePath, std::list() }); + } + vMap[svExtension][svFilePath].push_back(vBlock); + } + + for (auto& iKeyValue : vMap) + { + writer.WriteString(iKeyValue.first); + for (auto& jKeyValue : iKeyValue.second) + { + writer.WriteString(jKeyValue.first); + for (auto& eKeyValue : jKeyValue.second) + { + writer.WriteString(GetFileName(eKeyValue.m_svBlockPath, true)); + {/*Write entry block*/ + writer.Write(eKeyValue.m_nCrc32); + writer.Write(eKeyValue.m_nPreloadBytes); + writer.Write(eKeyValue.m_iArchiveIndex); + + for (size_t i = 0; i < eKeyValue.m_vvEntries.size(); i++) + { + {/*Write entry descriptor*/ + writer.Write(eKeyValue.m_vvEntries[i].m_nEntryFlags); + writer.Write(eKeyValue.m_vvEntries[i].m_nTextureFlags); + writer.Write(eKeyValue.m_vvEntries[i].m_nArchiveOffset); + writer.Write(eKeyValue.m_vvEntries[i].m_nCompressedSize); + writer.Write(eKeyValue.m_vvEntries[i].m_nUncompressedSize); + } + + if (i != (eKeyValue.m_vvEntries.size() - 1)) + { + const ushort s = 0; + writer.Write(s); + } + else + { + const ushort s = UINT16_MAX; + writer.Write(s); + } + } + } + } + writer.Write('\0'); + } + writer.Write('\0'); + } + writer.Write('\0'); + m_vHeader.m_nTreeSize = static_cast(writer.GetPosition() - sizeof(VPKHeader_t)); + + writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nTreeSize)); + writer.Write(this->m_vHeader.m_nTreeSize); + writer.Write(0); +} /////////////////////////////////////////////////////////////////////////////// CPackedStore* g_pPackedStore = new CPackedStore();