/*********************************************************************** * ██████╗ ██████╗ ██╗ ██╗██████╗ ██╗ ██╗ ██╗ ██╗██████╗ * * ██╔══██╗╚════██╗ ██║ ██║██╔══██╗██║ ██╔╝ ██║ ██║██╔══██╗ * * ██████╔╝ █████╔╝ ██║ ██║██████╔╝█████╔╝ ██║ ██║██████╔╝ * * ██╔══██╗██╔═══╝ ╚██╗ ██╔╝██╔═══╝ ██╔═██╗ ██║ ██║██╔══██╗ * * ██║ ██║███████╗ ╚████╔╝ ██║ ██║ ██╗ ███████╗██║██████╔╝ * * ╚═╝ ╚═╝╚══════╝ ╚═══╝ ╚═╝ ╚═╝ ╚═╝ ╚══════╝╚═╝╚═════╝ * ***********************************************************************/ #include "core/stdafx.h" #include "tier1/cvar.h" #include "mathlib/adler32.h" #include "mathlib/crc32.h" #include "vpklib/packedstore.h" //----------------------------------------------------------------------------- // Purpose: initialize parameters for compression algorithm //----------------------------------------------------------------------------- void CPackedStore::InitLzCompParams(void) { /*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ m_lzCompParams.m_dict_size_log2 = RVPK_DICT_SIZE; m_lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_FASTER; //m_lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING | lzham_compress_flags::LZHAM_COMP_FLAG_TRADEOFF_DECOMPRESSION_RATE_FOR_COMP_RATIO; m_lzCompParams.m_max_helper_threads = -1; } //----------------------------------------------------------------------------- // Purpose: initialize parameters for decompression algorithm //----------------------------------------------------------------------------- void CPackedStore::InitLzDecompParams(void) { /*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ m_lzDecompParams.m_dict_size_log2 = RVPK_DICT_SIZE; m_lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED | lzham_decompress_flags::LZHAM_DECOMP_FLAG_COMPUTE_CRC32; m_lzDecompParams.m_struct_size = sizeof(lzham_decompress_params); } //----------------------------------------------------------------------------- // Purpose: obtains archive chunk path for specific file //----------------------------------------------------------------------------- string CPackedStore::GetPackChunkFile(string svPackDirFile, int iArchiveIndex) { /*| ARCHIVES ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ string svPackChunkFile = StripLocalePrefix(svPackDirFile); ostringstream oss; oss << std::setw(3) << std::setfill('0') << iArchiveIndex; string svPackChunkIndex = "pak000_" + oss.str(); StringReplace(svPackChunkFile, "pak000_dir", svPackChunkIndex); return svPackChunkFile; } //----------------------------------------------------------------------------- // Purpose: returns populated pack dir struct for specified pack dir file //----------------------------------------------------------------------------- VPKDir_t CPackedStore::GetPackDirFile(string svPackDirFile) { /*| PACKDIRFILE |||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ std::regex rgArchiveRegex("pak000_([0-9]{3})"); std::smatch smRegexMatches; std::regex_search(svPackDirFile, smRegexMatches, rgArchiveRegex); if (smRegexMatches.size() != 0) { StringReplace(svPackDirFile, smRegexMatches[0], "pak000_dir"); for (int i = 0; i < LANGUAGE_PACKS; i++) { if (strstr(svPackDirFile.c_str(), DIR_LIBRARY_PREFIX[i].c_str())) { for (int j = 0; j < LIBRARY_PACKS; j++) { if (strstr(svPackDirFile.c_str(), DIR_LIBRARY_PREFIX[j].c_str())) { string svPackDirPrefix = DIR_LOCALE_PREFIX[i] + DIR_LOCALE_PREFIX[i]; StringReplace(svPackDirFile, DIR_LOCALE_PREFIX[i].c_str(), svPackDirPrefix.c_str()); goto escape; } } } }escape:; } VPKDir_t vpk_dir(svPackDirFile); return vpk_dir; } //----------------------------------------------------------------------------- // Purpose: obtains and returns the entry block to the vector //----------------------------------------------------------------------------- vector CPackedStore::GetEntryBlocks(CIOStream* pReader) { /*| ENTRYBLOCKS |||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ string svName, svPath, svExtension; vector vBlocks; while (!(svExtension = pReader->ReadString()).empty()) { while (!(svPath = pReader->ReadString()).empty()) { while (!(svName = pReader->ReadString()).empty()) { string svFilePath = FormatBlockPath(svName, svPath, svExtension); vBlocks.push_back(VPKEntryBlock_t(pReader, svFilePath)); } } } return vBlocks; } //----------------------------------------------------------------------------- // Purpose: scans the input directory and returns the paths to the vector //----------------------------------------------------------------------------- vector CPackedStore::GetEntryPaths(const string& svPathIn) const { vector vPaths; for (const fs::directory_entry& dirEntry : fs::recursive_directory_iterator(fs_packedstore_workspace->GetString())) { if (!GetExtension(dirEntry.path().u8string()).empty()) { vPaths.push_back(ConvertToUnixPath(dirEntry.path().u8string())); } } return vPaths; } //----------------------------------------------------------------------------- // Purpose: formats the entry block path //----------------------------------------------------------------------------- string CPackedStore::FormatBlockPath(string svName, string svPath, string svExtension) { if (!svPath.empty()) { svPath += "\\"; } return svPath + svName + "." + svExtension; } //----------------------------------------------------------------------------- // Purpose: strips locale prefix from file path //----------------------------------------------------------------------------- string CPackedStore::StripLocalePrefix(string svPackDirFile) { fs::path fspPackDirFile(svPackDirFile); string svFileName = fspPackDirFile.filename().u8string(); for (int i = 0; i < LANGUAGE_PACKS; i++) { if (strstr(svFileName.c_str(), DIR_LOCALE_PREFIX[i].c_str())) { StringReplace(svFileName, DIR_LOCALE_PREFIX[i].c_str(), ""); break; } } return svFileName; } //----------------------------------------------------------------------------- // Purpose: validates extraction result with precomputed ADLER32 hash //----------------------------------------------------------------------------- void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile) { CIOStream reader(svAssetFile, CIOStream::Mode_t::READ); m_nAdler32 = adler32::update(m_nAdler32, reader.GetData(), reader.GetSize()); if (m_nAdler32 != m_nAdler32_Internal) { Warning(eDLL_T::FS, "Warning: ADLER32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nAdler32, m_nAdler32_Internal); m_nAdler32 = 0; m_nAdler32_Internal = 0; } } //----------------------------------------------------------------------------- // Purpose: validates extraction result with precomputed CRC32 hash //----------------------------------------------------------------------------- void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile) { CIOStream reader(svAssetFile, CIOStream::Mode_t::READ); m_nCrc32 = crc32::update(m_nCrc32, reader.GetData(), reader.GetSize()); if (m_nCrc32 != m_nCrc32_Internal) { Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nCrc32, m_nCrc32_Internal); m_nCrc32 = 0; m_nCrc32_Internal = 0; } } void CPackedStore::PackAll(string svDirIn, string svPathOut) { CIOStream writer("client_mp_rr_canyonlands_staging.bsp.pak000_000.vpk", CIOStream::Mode_t::WRITE); vector vEntryBlocks; vector vPaths = GetEntryPaths(svDirIn); for (size_t i = 0; i < vPaths.size(); i++) { CIOStream reader(vPaths[i], CIOStream::Mode_t::READ); if (reader.IsReadable()) { vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), 0, 0x101, 0, StringReplaceC(vPaths[i], fs_packedstore_workspace->GetString(), ""))); for (size_t j = 0; j < vEntryBlocks[i].m_vvEntries.size(); j++) { uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; uint8_t* pDest = new uint8_t[DECOMP_MAX_BUF]; reader.Read(*pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); vEntryBlocks[i].m_vvEntries[j].m_nArchiveOffset = writer.GetPosition(); m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize, pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) { Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), vEntryBlocks.at(i).m_iArchiveIndex); Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d' (file will be packed without compression.)\n", m_lzCompStatus); vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize = vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; writer.Write(pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); } else writer.Write(pDest, vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize); vEntryBlocks[i].m_vvEntries[j].m_bIsCompressed = vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize != vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; delete[] pSrc; delete[] pDest; } } } VPKDir_t vDir = VPKDir_t(); vDir.Build("englishclient_mp_rr_canyonlands_staging.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <> !!!] } //----------------------------------------------------------------------------- // Purpose: extracts all files from specified vpk file //----------------------------------------------------------------------------- void CPackedStore::UnpackAll(VPKDir_t vpkDir, string svPathOut) { for (int i = 0; i < vpkDir.m_vsvArchives.size(); i++) { fs::path fspVpkPath(vpkDir.m_svDirPath); string svPath = fspVpkPath.parent_path().u8string() + "\\" + vpkDir.m_vsvArchives[i]; ifstream packChunkStream(svPath, std::ios_base::binary); // Create stream to read from each archive. for ( VPKEntryBlock_t block : vpkDir.m_vvEntryBlocks) { // Escape if block archive index is not part of the extracting archive chunk index. if (block.m_iArchiveIndex != i) { goto escape; } else { string svFilePath = CreateDirectories(svPathOut + "\\" + block.m_svBlockPath); ofstream outFileStream(svFilePath, std::ios_base::binary | std::ios_base::out); if (!outFileStream.is_open()) { Error(eDLL_T::FS, "Error: unable to access file '%s'!\n", svFilePath.c_str()); } outFileStream.clear(); // Make sure file is empty before writing. for (VPKEntryDescriptor_t entry : block.m_vvEntries) { char* pCompressedData = new char[entry.m_nCompressedSize]; memset(pCompressedData, 0, entry.m_nCompressedSize); // Compressed region. packChunkStream.seekg(entry.m_nArchiveOffset); // Seek to entry offset in archive. packChunkStream.read(pCompressedData, entry.m_nCompressedSize); // Read compressed data from archive. if (entry.m_bIsCompressed) { lzham_uint8* pLzOutputBuf = new lzham_uint8[entry.m_nUncompressedSize]; m_lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, pLzOutputBuf, (size_t*)&entry.m_nUncompressedSize, (lzham_uint8*)pCompressedData, entry.m_nCompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); if (fs_packedstore_entryblock_stats->GetBool()) { DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); DevMsg(eDLL_T::FS, "] Block path : '%s'\n", block.m_svBlockPath.c_str()); DevMsg(eDLL_T::FS, "] Entry count : '%llu'\n", block.m_vvEntries.size()); DevMsg(eDLL_T::FS, "] Compressed size : '%llu'\n", entry.m_nCompressedSize); DevMsg(eDLL_T::FS, "] Uncompressed size : '%llu'\n", entry.m_nUncompressedSize); DevMsg(eDLL_T::FS, "] Static CRC32 hash : '0x%lX'\n", block.m_nCrc32); DevMsg(eDLL_T::FS, "] Computed CRC32 hash : '0x%lX'\n", m_nCrc32_Internal); DevMsg(eDLL_T::FS, "] Computed ADLER32 hash : '0x%lX'\n", m_nAdler32_Internal); DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); } if (block.m_vvEntries.size() == 1) // Internal checksum can only match block checksum if entry size is 1. { if (block.m_nCrc32 != m_nCrc32_Internal) { Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", block.m_svBlockPath.c_str(), m_nCrc32_Internal, block.m_nCrc32); } } else { m_nEntryCount++; } if (m_lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS) { Error(eDLL_T::FS, "Error: failed decompression for an entry within block '%s' in archive '%d'!\n", block.m_svBlockPath.c_str(), i); Error(eDLL_T::FS, "'lzham::lzham_lib_decompress_memory' returned with status '%d'.\n", m_lzDecompStatus); } else { // If successfully decompressed, write to file. outFileStream.write((char*)pLzOutputBuf, entry.m_nUncompressedSize); } delete[] pLzOutputBuf; } else { // If not compressed, write raw data into output file. outFileStream.write(pCompressedData, entry.m_nUncompressedSize); } delete[] pCompressedData; } outFileStream.close(); if (m_nEntryCount == block.m_vvEntries.size()) // Only validate after last entry in block had been written. { // Set internal hash to precomputed entry hash for post decompress validation. m_nCrc32_Internal = block.m_nCrc32; ValidateCRC32PostDecomp(svFilePath); //ValidateAdler32PostDecomp(svFilePath); m_nEntryCount = 0; } }escape:; } packChunkStream.close(); } } //----------------------------------------------------------------------------- // Purpose: 'vpk_entry_block' constructor //----------------------------------------------------------------------------- VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* reader, string svPath) { std::replace(svPath.begin(), svPath.end(), '/', '\\'); // Flip forward slashes in filepath to windows-style backslash. this->m_svBlockPath = svPath; // Set path of block. reader->Read(this->m_nCrc32); // reader->Read(this->m_nPreloadBytes); // reader->Read(this->m_iArchiveIndex); // do // Loop through all entries in the block and push them to the vector. { VPKEntryDescriptor_t entry(reader); this->m_vvEntries.push_back(entry); } while (reader->Read() != 0xFFFF); } #undef min VPKEntryBlock_t::VPKEntryBlock_t(const vector &vData, int64_t nOffset, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, string svBlockPath) { m_nCrc32 = crc32::update(m_nCrc32, vData.data(), vData.size()); m_nPreloadBytes = 0; m_iArchiveIndex = nArchiveIndex; m_svBlockPath = svBlockPath; int nEntryCount = (vData.size() + RVPK_MAX_BLOCK - 1) / RVPK_MAX_BLOCK; uint64_t nDataSize = vData.size(); int64_t nCurrentOffset = nOffset; for (int i = 0; i < nEntryCount; i++) { uint64_t nSize = std::min(RVPK_MAX_BLOCK, nDataSize); nDataSize -= nSize; m_vvEntries.push_back(VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize)); nCurrentOffset += nSize; } } VPKEntryDescriptor_t::VPKEntryDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nArchiveOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize) { m_nEntryFlags = nEntryFlags; m_nTextureFlags = nTextureFlags; m_nArchiveOffset = nArchiveOffset; m_nCompressedSize = nCompressedSize; m_nUncompressedSize = nUncompressedSize; } //----------------------------------------------------------------------------- // Purpose: 'VPKDir_t' constructor // Input : *pReader - //----------------------------------------------------------------------------- VPKEntryDescriptor_t::VPKEntryDescriptor_t(CIOStream* pReader) { pReader->Read(this->m_nEntryFlags); // pReader->Read(this->m_nTextureFlags); // pReader->Read(this->m_nArchiveOffset); // pReader->Read(this->m_nCompressedSize); // pReader->Read(this->m_nUncompressedSize); // this->m_bIsCompressed = (this->m_nCompressedSize != this->m_nUncompressedSize); } //----------------------------------------------------------------------------- // Purpose: 'VPKDir_t' file constructor //----------------------------------------------------------------------------- VPKDir_t::VPKDir_t(const string& svPath) { CIOStream reader(svPath, CIOStream::Mode_t::READ); reader.Read(this->m_vHeader.m_nFileMagic); if (this->m_vHeader.m_nFileMagic != RVPK_DIR_MAGIC) { Error(eDLL_T::FS, "Error: vpk_dir file '%s' has invalid magic!\n", svPath.c_str()); //return; } reader.Read(this->m_vHeader.m_nMajorVersion); // reader.Read(this->m_vHeader.m_nMinorVersion); // reader.Read(this->m_vHeader.m_nTreeSize); // reader.Read(this->m_nFileDataSize); // DevMsg(eDLL_T::FS, "______________________________________________________________\n"); DevMsg(eDLL_T::FS, "] HEADER_DETAILS ---------------------------------------------\n"); DevMsg(eDLL_T::FS, "] File Magic : '%lu'\n", this->m_vHeader.m_nFileMagic); DevMsg(eDLL_T::FS, "] Major Version : '%hu'\n", this->m_vHeader.m_nMajorVersion); DevMsg(eDLL_T::FS, "] Minor Version : '%hu'\n", this->m_vHeader.m_nMinorVersion); DevMsg(eDLL_T::FS, "] Tree Size : '%lu'\n", this->m_vHeader.m_nTreeSize); DevMsg(eDLL_T::FS, "] File Data Size : '%lu'\n", this->m_nFileDataSize); this->m_vvEntryBlocks = g_pPackedStore->GetEntryBlocks(&reader); this->m_svDirPath = svPath; // Set path to vpk_dir file. for (VPKEntryBlock_t block : this->m_vvEntryBlocks) { if (block.m_iArchiveIndex > this->m_iArchiveCount) { this->m_iArchiveCount = block.m_iArchiveIndex; } } DevMsg(eDLL_T::FS, "______________________________________________________________\n"); DevMsg(eDLL_T::FS, "] PACK_CHUNKS ------------------------------------------------\n"); for (int i = 0; i < this->m_iArchiveCount + 1; i++) { string svArchivePath = g_pPackedStore->GetPackChunkFile(svPath, i); DevMsg(eDLL_T::FS, "] '%s'\n", svArchivePath.c_str()); this->m_vsvArchives.push_back(svArchivePath); } } //----------------------------------------------------------------------------- // Purpose: builds the VPKDir file // Input : &svFileName - // &vEntryBlocks - //----------------------------------------------------------------------------- void VPKDir_t::Build(const string& svFileName, const vector& vEntryBlocks) { CIOStream writer(svFileName, CIOStream::Mode_t::WRITE); auto vMap = std::map>>(); writer.Write(this->m_vHeader.m_nFileMagic); writer.Write(this->m_vHeader.m_nMajorVersion); writer.Write(this->m_vHeader.m_nMinorVersion); writer.Write(this->m_vHeader.m_nTreeSize); writer.Write(this->m_vHeader.m_nTreeSize); for (VPKEntryBlock_t vBlock : vEntryBlocks) { string svExtension = GetExtension(vBlock.m_svBlockPath); string svFileName = GetFileName(vBlock.m_svBlockPath, true); string svFilePath = RemoveFileName(vBlock.m_svBlockPath); if (svFilePath.empty()) { svFilePath = " "; // Has to be padded with a space character if empty. } if (!vMap.count(svExtension)) { vMap.insert({ svExtension, std::map>() }); } if (!vMap[svExtension].count(svFilePath)) { vMap[svExtension].insert({ svFilePath, std::list() }); } vMap[svExtension][svFilePath].push_back(vBlock); } for (auto& iKeyValue : vMap) { writer.WriteString(iKeyValue.first); for (auto& jKeyValue : iKeyValue.second) { writer.WriteString(jKeyValue.first); for (auto& eKeyValue : jKeyValue.second) { writer.WriteString(GetFileName(eKeyValue.m_svBlockPath, true)); {/*Write entry block*/ writer.Write(eKeyValue.m_nCrc32); writer.Write(eKeyValue.m_nPreloadBytes); writer.Write(eKeyValue.m_iArchiveIndex); for (size_t i = 0; i < eKeyValue.m_vvEntries.size(); i++) { {/*Write entry descriptor*/ writer.Write(eKeyValue.m_vvEntries[i].m_nEntryFlags); writer.Write(eKeyValue.m_vvEntries[i].m_nTextureFlags); writer.Write(eKeyValue.m_vvEntries[i].m_nArchiveOffset); writer.Write(eKeyValue.m_vvEntries[i].m_nCompressedSize); writer.Write(eKeyValue.m_vvEntries[i].m_nUncompressedSize); } if (i != (eKeyValue.m_vvEntries.size() - 1)) { const ushort s = 0; writer.Write(s); } else { const ushort s = UINT16_MAX; writer.Write(s); } } } } writer.Write('\0'); } writer.Write('\0'); } writer.Write('\0'); m_vHeader.m_nTreeSize = static_cast(writer.GetPosition() - sizeof(VPKHeader_t)); writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nTreeSize)); writer.Write(this->m_vHeader.m_nTreeSize); writer.Write(0); } /////////////////////////////////////////////////////////////////////////////// CPackedStore* g_pPackedStore = new CPackedStore();