diff --git a/r5dev/common/sdkdefs.h b/r5dev/common/sdkdefs.h index f3ed5817..d3678d4e 100644 --- a/r5dev/common/sdkdefs.h +++ b/r5dev/common/sdkdefs.h @@ -17,6 +17,7 @@ using std::ostringstream; using std::unordered_map; namespace fs = std::filesystem; +namespace js = nlohmann; typedef const unsigned char* rsig_t; #endif // SDKDEFS_H diff --git a/r5dev/public/include/binstream.h b/r5dev/public/include/binstream.h index e29a4968..a4d39f29 100644 --- a/r5dev/public/include/binstream.h +++ b/r5dev/public/include/binstream.h @@ -37,10 +37,6 @@ public: { if (IsReadable()) m_iStream.read(reinterpret_cast(&tValue), sizeof(tValue)); - else - printf("TEST\n"); - - printf("%d\n", tValue); } //----------------------------------------------------------------------------- diff --git a/r5dev/public/include/utility.h b/r5dev/public/include/utility.h index 99a28ecc..05f01386 100644 --- a/r5dev/public/include/utility.h +++ b/r5dev/public/include/utility.h @@ -20,7 +20,7 @@ string RemoveExtension(const string& svInput); string GetFileName(const string& svInput, bool bRemoveExtension, bool bWindows = false); string RemoveFileName(const string& svInput, bool bWindows = false); -string CreateDirectories(string svInput); +string CreateDirectories(string svInput, bool bWindows = false); string ConvertToWinPath(const string& svInput); string ConvertToUnixPath(const string& svInput); diff --git a/r5dev/public/utility.cpp b/r5dev/public/utility.cpp index 11f0a4d1..cd917497 100644 --- a/r5dev/public/utility.cpp +++ b/r5dev/public/utility.cpp @@ -277,7 +277,7 @@ string GetFileName(const string& svInput, bool bRemoveExtension, bool bWindows) } return svInput.substr(nPos + 1); } - else // File name is not within path. + else // File name is not within a path. { if (bRemoveExtension) { @@ -309,14 +309,21 @@ string RemoveFileName(const string& svInput, bool bWindows) /////////////////////////////////////////////////////////////////////////////// // For creating directories for output streams. -string CreateDirectories(string svInput) +string CreateDirectories(string svInput, bool bWindows) { + if (bWindows) + { + StringReplace(svInput, "\\ \\", "\\"); + } + else + { + StringReplace(svInput, "/ /", "/"); + } + fs::path fspPathOut(svInput); string results = fspPathOut.u8string(); - StringReplace(svInput, "\\ \\", "\\"); fspPathOut = fspPathOut.parent_path(); - fs::create_directories(fspPathOut); return results; diff --git a/r5dev/vpklib/packedstore.cpp b/r5dev/vpklib/packedstore.cpp index c98bbb85..af3eeabc 100644 --- a/r5dev/vpklib/packedstore.cpp +++ b/r5dev/vpklib/packedstore.cpp @@ -38,8 +38,10 @@ void CPackedStore::InitLzDecompParams(void) //----------------------------------------------------------------------------- // Purpose: returns populated pack dir struct for specified pack dir file +// Input : svPackDirFile - +// Output : VPKDir_t //----------------------------------------------------------------------------- -VPKDir_t CPackedStore::GetPackDirFile(string svPackDirFile) +VPKDir_t CPackedStore::GetPackDirFile(string svPackDirFile) const { /*| PACKDIRFILE |||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ std::regex rgArchiveRegex("pak000_([0-9]{3})"); @@ -51,16 +53,16 @@ VPKDir_t CPackedStore::GetPackDirFile(string svPackDirFile) { StringReplace(svPackDirFile, smRegexMatches[0], "pak000_dir"); - for (int i = 0; i < LANGUAGE_PACKS; i++) + for (size_t i = 0; i < DIR_LOCALE.size(); i++) { - if (strstr(svPackDirFile.c_str(), DIR_LIBRARY_PREFIX[i].c_str())) + if (strstr(svPackDirFile.c_str(), DIR_CONTEXT[i].c_str())) { - for (int j = 0; j < LIBRARY_PACKS; j++) + for (size_t j = 0; j < DIR_CONTEXT.size(); j++) { - if (strstr(svPackDirFile.c_str(), DIR_LIBRARY_PREFIX[j].c_str())) + if (strstr(svPackDirFile.c_str(), DIR_CONTEXT[j].c_str())) { - string svPackDirPrefix = DIR_LOCALE_PREFIX[i] + DIR_LOCALE_PREFIX[i]; - StringReplace(svPackDirFile, DIR_LOCALE_PREFIX[i].c_str(), svPackDirPrefix.c_str()); + string svPackDirPrefix = DIR_LOCALE[i] + DIR_LOCALE[i]; + StringReplace(svPackDirFile, DIR_LOCALE[i].c_str(), svPackDirPrefix.c_str()); goto escape; } } @@ -74,8 +76,11 @@ VPKDir_t CPackedStore::GetPackDirFile(string svPackDirFile) //----------------------------------------------------------------------------- // Purpose: obtains archive chunk path for specific file +// Input : &svPackDirFile - +// iArchiveIndex - +// output : string //----------------------------------------------------------------------------- -string CPackedStore::GetPackChunkFile(const string& svPackDirFile, int iArchiveIndex) +string CPackedStore::GetPackChunkFile(const string& svPackDirFile, int iArchiveIndex) const { /*| ARCHIVES ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ string svPackChunkFile = StripLocalePrefix(svPackDirFile); @@ -90,8 +95,10 @@ string CPackedStore::GetPackChunkFile(const string& svPackDirFile, int iArchiveI //----------------------------------------------------------------------------- // Purpose: obtains and returns the entry block to the vector +// Input : *pReader - +// output : vector //----------------------------------------------------------------------------- -vector CPackedStore::GetEntryBlocks(CIOStream* pReader) +vector CPackedStore::GetEntryBlocks(CIOStream* pReader) const { /*| ENTRYBLOCKS |||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/ string svName, svPath, svExtension; @@ -112,53 +119,208 @@ vector CPackedStore::GetEntryBlocks(CIOStream* pReader) //----------------------------------------------------------------------------- // Purpose: scans the input directory and returns the paths to the vector +// Input : &svPathIn - +// Output : vector //----------------------------------------------------------------------------- vector CPackedStore::GetEntryPaths(const string& svPathIn) const { vector vPaths; - for (const fs::directory_entry& dirEntry : fs::recursive_directory_iterator(fs_packedstore_workspace->GetString())) + fs::recursive_directory_iterator dir(svPathIn), end; + while (dir != end) { - if (!GetExtension(dirEntry.path().u8string()).empty()) + if (dir->path().filename() == "manifest") { - vPaths.push_back(ConvertToUnixPath(dirEntry.path().u8string())); + dir.disable_recursion_pending(); // Don't recurse into this directory (manifest files only). } + if (!GetExtension(dir->path().u8string()).empty()) + { + vPaths.push_back(ConvertToUnixPath(dir->path().u8string())); + } + dir++; } return vPaths; } //----------------------------------------------------------------------------- -// Purpose: formats the entry block path +// Purpose: scans the input directory and returns the paths to the vector if path exists in manifest +// Input : &svPathIn - +// &jManifest - +// Output : vector //----------------------------------------------------------------------------- -string CPackedStore::FormatBlockPath(string svPath, const string& svName, const string& svExtension) +vector CPackedStore::GetEntryPaths(const string& svPathIn, const js::json& jManifest) const +{ + vector vPaths; + fs::recursive_directory_iterator dir(svPathIn), end; + while (dir != end) + { + if (dir->path().filename() == "manifest") + { + dir.disable_recursion_pending(); // Don't recurse into this directory (manifest files only). + } + if (!GetExtension(dir->path().u8string()).empty()) + { + if (!jManifest.is_null()) + { + try + { + string svBlockPath = ConvertToUnixPath(dir->path().u8string()); + if (jManifest.contains(StringReplaceC(svBlockPath, svPathIn, ""))) + { + printf("%s\n", svBlockPath.c_str()); + vPaths.push_back(svBlockPath); + } + } + catch (const std::exception& ex) + { + Warning(eDLL_T::FS, "Exception while reading VPK manifest file: '%s'\n", ex.what()); + } + } + } + dir++; + } + return vPaths; +} + +//----------------------------------------------------------------------------- +// Purpose: gets the raw level name from the directory file name +// Input : &svDirectoryName - +// Output : string +//----------------------------------------------------------------------------- +string CPackedStore::GetLevelName(const string& svDirectoryName) const +{ + std::regex rgArchiveRegex{ R"([^_]*_(.*)(.bsp.pak000_dir).*)" }; + std::smatch smRegexMatches; + std::regex_search(svDirectoryName, smRegexMatches, rgArchiveRegex); + + return smRegexMatches[1].str(); +} + +//----------------------------------------------------------------------------- +// Purpose: gets the manifest file assosiated with the VPK name +// Input : &svManifestName - +// Output : json +//----------------------------------------------------------------------------- +js::json CPackedStore::GetManifest(const string& svWorkSpace, const string& svManifestName) const +{ + ostringstream ostream; + ostream << svWorkSpace << "manifest/" << svManifestName << ".json"; + fs::path fsPath = fs::current_path() /= ostream.str(); + + + printf("%s\n", fsPath.string().c_str()); + + if (fs::exists(fsPath)) + { + js::json jsOut; + try + { + ifstream iManifest(fsPath.string().c_str(), std::ios::binary); + jsOut = js::json::parse(iManifest); + + return jsOut; + } + catch (const std::exception& ex) + { + Warning(eDLL_T::FS, "Exception while parsing VPK manifest file: '%s'\n", ex.what()); + return jsOut; + } + } +} + +//----------------------------------------------------------------------------- +// Purpose: formats the entry block path +// Input : svPath - +// &svName - +// &svExtension - +// Output : string +//----------------------------------------------------------------------------- +string CPackedStore::FormatBlockPath(string svPath, const string& svName, const string& svExtension) const { if (!svPath.empty()) { - svPath += "\\"; + svPath += '/'; } - return svPath + svName + "." + svExtension; + return svPath + svName + '.' + svExtension; } //----------------------------------------------------------------------------- // Purpose: strips locale prefix from file path +// Input : &svDirectoryFile - +// Output : string //----------------------------------------------------------------------------- -string CPackedStore::StripLocalePrefix(const string& svPackDirFile) +string CPackedStore::StripLocalePrefix(const string& svDirectoryFile) const { - fs::path fspPackDirFile(svPackDirFile); - string svFileName = fspPackDirFile.filename().u8string(); + fs::path fsDirectoryFile(svDirectoryFile); + string svFileName = fsDirectoryFile.filename().u8string(); - for (int i = 0; i < LANGUAGE_PACKS; i++) + for (size_t i = 0; i < DIR_LOCALE.size(); i++) { - if (strstr(svFileName.c_str(), DIR_LOCALE_PREFIX[i].c_str())) + if (strstr(svFileName.c_str(), DIR_LOCALE[i].c_str())) { - StringReplace(svFileName, DIR_LOCALE_PREFIX[i].c_str(), ""); + StringReplace(svFileName, DIR_LOCALE[i].c_str(), ""); break; } } return svFileName; } +//----------------------------------------------------------------------------- +// Purpose: builds a valid file name for the VPK +// Input : svLanguage - +// svContext - +// &svPakName - +// nPatch - +// Output : VPKPair_t +//----------------------------------------------------------------------------- +VPKPair_t CPackedStore::BuildFileName(string svLanguage, string svContext, const string& svPakName, int nPatch) const +{ + if (std::find(DIR_LOCALE.begin(), DIR_LOCALE.end(), svLanguage) == DIR_LOCALE.end()) + { + svLanguage = DIR_LOCALE[0]; + } + if (std::find(DIR_CONTEXT.begin(), DIR_CONTEXT.end(), svContext) == DIR_CONTEXT.end()) + { + svContext = DIR_CONTEXT[0]; + } + + VPKPair_t vPair; + vPair.m_svBlockName = fmt::format("{:s}_{:s}.bsp.pak000_{:03d}{:s}", svContext, svPakName, nPatch, ".vpk"); + vPair.m_svDirectoryName = fmt::format("{:s}{:s}_{:s}.bsp.pak000_{:s}", svLanguage, svContext, svPakName, "dir.vpk"); + + return vPair; +} + +//----------------------------------------------------------------------------- +// Purpose: builds the VPK manifest file +// Input : &vBlock - +// &svOutPath - +// Output : VPKPair_t +//----------------------------------------------------------------------------- +void CPackedStore::BuildManifest(const vector& vBlock, const string& svWorkSpace, const string& svManifestName) const +{ + js::json jEntry; + + for (size_t i = 0; i < vBlock.size(); i++) + { + jEntry[vBlock[i].m_svBlockPath] = + { + { "preloadData", vBlock[i].m_nPreloadData}, + { "entryFlags", vBlock[i].m_vvEntries[0].m_nEntryFlags}, + { "textureFlags", vBlock[i].m_vvEntries[0].m_nTextureFlags}, + { "useCompression", vBlock[i].m_vvEntries[0].m_nCompressedSize != vBlock[i].m_vvEntries[0].m_nUncompressedSize} + }; + } + + string svPathOut = svWorkSpace + "manifest\\"; + fs::create_directories(svPathOut); + + ofstream oManifest(svPathOut + svManifestName + ".json"); + oManifest << jEntry.dump(4); +} + //----------------------------------------------------------------------------- // Purpose: validates extraction result with precomputed ADLER32 hash +// Input : &svAssetFile - //----------------------------------------------------------------------------- void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile) { @@ -167,14 +329,15 @@ void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetFile) if (m_nAdler32 != m_nAdler32_Internal) { - Warning(eDLL_T::FS, "Warning: ADLER32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nAdler32, m_nAdler32_Internal); - m_nAdler32 = 0; - m_nAdler32_Internal = 0; + Warning(eDLL_T::FS, "ADLER32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nAdler32, m_nAdler32_Internal); + m_nAdler32 = NULL; + m_nAdler32_Internal = NULL; } } //----------------------------------------------------------------------------- // Purpose: validates extraction result with precomputed CRC32 hash +// Input : &svAssetFile - //----------------------------------------------------------------------------- void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile) { @@ -183,66 +346,120 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetFile) if (m_nCrc32 != m_nCrc32_Internal) { - Warning(eDLL_T::FS, "Warning: CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nCrc32, m_nCrc32_Internal); - m_nCrc32 = 0; - m_nCrc32_Internal = 0; + Warning(eDLL_T::FS, "CRC32 checksum mismatch for entry '%s' computed value '0x%lX' doesn't match expected value '0x%lX'. File may be corrupt!\n", svAssetFile.c_str(), m_nCrc32, m_nCrc32_Internal); + m_nCrc32 = NULL; + m_nCrc32_Internal = NULL; } } -void CPackedStore::PackAll(const string& svDirIn, const string& svPathOut) +//----------------------------------------------------------------------------- +// Purpose: packs all files from specified path into VPK file +// Input : &vPair - +// &svPathIn - +// &svPathOut - +// bManifestOnly - +//----------------------------------------------------------------------------- +void CPackedStore::PackAll(const VPKPair_t& vPair, const string& svPathIn, const string& svPathOut, bool bManifestOnly) { - CIOStream writer("client_mp_rr_canyonlands_staging.bsp.pak000_000.vpk", CIOStream::Mode_t::WRITE); + CIOStream writer(svPathOut + vPair.m_svBlockName, CIOStream::Mode_t::WRITE); + string svLevelName = GetLevelName(vPair.m_svDirectoryName); + vector vPaths; vector vEntryBlocks; - vector vPaths = GetEntryPaths(svDirIn); + js::json jManifest = GetManifest(svPathIn, svLevelName); + + if (bManifestOnly) + { + vPaths = GetEntryPaths(svPathIn, jManifest); + } + else // Compress all files in workspace. + { + vPaths = GetEntryPaths(svPathIn); + } for (size_t i = 0; i < vPaths.size(); i++) { CIOStream reader(vPaths[i], CIOStream::Mode_t::READ); if (reader.IsReadable()) { - vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), 0, 0x101, 0, StringReplaceC(vPaths[i], fs_packedstore_workspace->GetString(), ""))); + uint16_t nPreloadData = 0i16; + uint32_t nEntryFlags = static_cast(EPackedEntryFlags::ENTRY_VISIBLE) | static_cast(EPackedEntryFlags::ENTRY_CACHE); + uint16_t nTextureFlags = static_cast(EPackedTextureFlags::TEXTURE_DEFAULT); // !TODO: Reverse these. + bool bUseCompression = true; + + if (!jManifest.is_null()) + { + try + { + js::json jEntry = jManifest[StringReplaceC(vPaths[i], svPathIn, "")]; + if (!jEntry.is_null()) + { + nPreloadData = jEntry.at("preloadData").get(); + nEntryFlags = jEntry.at("entryFlags").get(); + nTextureFlags = jEntry.at("textureFlags").get(); + bUseCompression = jEntry.at("useCompression").get(); + } + } + catch (const std::exception& ex) + { + Warning(eDLL_T::FS, "Exception while reading VPK manifest file: '%s'\n", ex.what()); + } + } + + vEntryBlocks.push_back(VPKEntryBlock_t(reader.GetVector(), writer.GetPosition(), nPreloadData, 0, nEntryFlags, nTextureFlags, StringReplaceC(vPaths[i], svPathIn, ""))); for (size_t j = 0; j < vEntryBlocks[i].m_vvEntries.size(); j++) { uint8_t* pSrc = new uint8_t[vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize]; - uint8_t* pDest = new uint8_t[DECOMP_MAX_BUF]; + uint8_t* pDest = new uint8_t[COMP_MAX];; reader.Read(*pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); vEntryBlocks[i].m_vvEntries[j].m_nArchiveOffset = writer.GetPosition(); - m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize, pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); - if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) + if (bUseCompression) { - Error(eDLL_T::FS, "Error: failed compression for an entry within block '%s' for archive '%d'\n", vEntryBlocks.at(i).m_svBlockPath.c_str(), vEntryBlocks.at(i).m_iArchiveIndex); - Error(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d' (file will be packed without compression.)\n", m_lzCompStatus); + m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, pDest, &vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize, pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); + if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS) + { + Warning(eDLL_T::FS, "Failed compression for entry '%d' within block '%s' for archive '%d'\n", j, vEntryBlocks.at(i).m_svBlockPath.c_str(), vEntryBlocks.at(i).m_iArchiveIndex); + Warning(eDLL_T::FS, "'lzham::lzham_lib_compress_memory' returned with status '%d' (entry will be packed without compression).\n", m_lzCompStatus); - vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize = vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; + vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize = vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; + writer.Write(pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); + } + else + { + writer.Write(pDest, vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize); + } + } + else // Write data uncompressed. + { writer.Write(pSrc, vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize); } - else - writer.Write(pDest, vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize); - vEntryBlocks[i].m_vvEntries[j].m_bIsCompressed = vEntryBlocks[i].m_vvEntries[j].m_nCompressedSize != vEntryBlocks[i].m_vvEntries[j].m_nUncompressedSize; - delete[] pSrc; delete[] pDest; + delete[] pSrc; } } } VPKDir_t vDir = VPKDir_t(); - vDir.Build("englishclient_mp_rr_canyonlands_staging.bsp.pak000_dir.vpk", vEntryBlocks); // [!!! <> !!!] + vDir.Build(svPathOut + vPair.m_svDirectoryName, vEntryBlocks); } //----------------------------------------------------------------------------- -// Purpose: extracts all files from specified vpk file +// Purpose: extracts all files from specified VPK file +// Input : &vpkDir - +// &svPathOut - //----------------------------------------------------------------------------- -void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) +void CPackedStore::UnpackAll(const VPKDir_t& vpkDir, const string& svPathOut) { + BuildManifest(vpkDir.m_vvEntryBlocks, svPathOut, GetLevelName(vpkDir.m_svDirPath)); + for (int i = 0; i < vpkDir.m_vsvArchives.size(); i++) { fs::path fspVpkPath(vpkDir.m_svDirPath); - string svPath = fspVpkPath.parent_path().u8string() + "\\" + vpkDir.m_vsvArchives[i]; + string svPath = fspVpkPath.parent_path().u8string() + '\\' + vpkDir.m_vsvArchives[i]; ifstream packChunkStream(svPath, std::ios_base::binary); // Create stream to read from each archive. for ( VPKEntryBlock_t block : vpkDir.m_vvEntryBlocks) @@ -251,7 +468,7 @@ void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) if (block.m_iArchiveIndex != i) { goto escape; } else { - string svFilePath = CreateDirectories(svPathOut + "\\" + block.m_svBlockPath); + string svFilePath = CreateDirectories(svPathOut + block.m_svBlockPath, true); ofstream outFileStream(svFilePath, std::ios_base::binary | std::ios_base::out); if (!outFileStream.is_open()) @@ -259,10 +476,11 @@ void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) Error(eDLL_T::FS, "Error: unable to access file '%s'!\n", svFilePath.c_str()); } outFileStream.clear(); // Make sure file is empty before writing. + for (VPKEntryDescriptor_t entry : block.m_vvEntries) { char* pCompressedData = new char[entry.m_nCompressedSize]; - memset(pCompressedData, 0, entry.m_nCompressedSize); // Compressed region. + memset(pCompressedData, '\0', entry.m_nCompressedSize); // Compressed region. packChunkStream.seekg(entry.m_nArchiveOffset); // Seek to entry offset in archive. packChunkStream.read(pCompressedData, entry.m_nCompressedSize); // Read compressed data from archive. @@ -274,19 +492,6 @@ void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) (size_t*)&entry.m_nUncompressedSize, (lzham_uint8*)pCompressedData, entry.m_nCompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal); - if (fs_packedstore_entryblock_stats->GetBool()) - { - DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); - DevMsg(eDLL_T::FS, "] Block path : '%s'\n", block.m_svBlockPath.c_str()); - DevMsg(eDLL_T::FS, "] Entry count : '%llu'\n", block.m_vvEntries.size()); - DevMsg(eDLL_T::FS, "] Compressed size : '%llu'\n", entry.m_nCompressedSize); - DevMsg(eDLL_T::FS, "] Uncompressed size : '%llu'\n", entry.m_nUncompressedSize); - DevMsg(eDLL_T::FS, "] Static CRC32 hash : '0x%lX'\n", block.m_nCrc32); - DevMsg(eDLL_T::FS, "] Computed CRC32 hash : '0x%lX'\n", m_nCrc32_Internal); - DevMsg(eDLL_T::FS, "] Computed ADLER32 hash : '0x%lX'\n", m_nAdler32_Internal); - DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); - } - if (block.m_vvEntries.size() == 1) // Internal checksum can only match block checksum if entry size is 1. { if (block.m_nCrc32 != m_nCrc32_Internal) @@ -298,7 +503,7 @@ void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) if (m_lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS) { - Error(eDLL_T::FS, "Error: failed decompression for an entry within block '%s' in archive '%d'!\n", block.m_svBlockPath.c_str(), i); + Error(eDLL_T::FS, "Failed decompression for an entry within block '%s' in archive '%d'!\n", block.m_svBlockPath.c_str(), i); Error(eDLL_T::FS, "'lzham::lzham_lib_decompress_memory' returned with status '%d'.\n", m_lzDecompStatus); } else @@ -315,6 +520,7 @@ void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) } delete[] pCompressedData; } + outFileStream.close(); if (m_nEntryCount == block.m_vvEntries.size()) // Only validate after last entry in block had been written. { @@ -332,43 +538,63 @@ void CPackedStore::UnpackAll(VPKDir_t vpkDir, const string& svPathOut) } //----------------------------------------------------------------------------- -// Purpose: 'vpk_entry_block' constructor +// Purpose: 'VPKEntryBlock_t' file constructor +// Input : *pReader - +// svBlockPath - //----------------------------------------------------------------------------- -VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* reader, string svPath) +VPKEntryBlock_t::VPKEntryBlock_t(CIOStream* pReader, string svBlockPath) { - std::replace(svPath.begin(), svPath.end(), '/', '\\'); // Flip forward slashes in filepath to windows-style backslash. + std::replace(svBlockPath.begin(), svBlockPath.end(), '\\', '/'); // Flip windows-style backslash to forward slash. - this->m_svBlockPath = svPath; // Set path of block. - reader->Read(this->m_nCrc32); // - reader->Read(this->m_nPreloadBytes); // - reader->Read(this->m_iArchiveIndex); // + this->m_svBlockPath = svBlockPath; // Set path of block. + pReader->Read(this->m_nCrc32); // + pReader->Read(this->m_nPreloadData); // + pReader->Read(this->m_iArchiveIndex); // do // Loop through all entries in the block and push them to the vector. { - VPKEntryDescriptor_t entry(reader); + VPKEntryDescriptor_t entry(pReader); this->m_vvEntries.push_back(entry); - } while (reader->Read() != 0xFFFF); + } while (pReader->Read() != UINT16_MAX); } -#undef min -VPKEntryBlock_t::VPKEntryBlock_t(const vector &vData, int64_t nOffset, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, string svBlockPath) + +//----------------------------------------------------------------------------- +// Purpose: 'VPKEntryBlock_t' memory constructor +// Input : &vData - +// nOffset - +// nPreloadData - +// nArchiveIndex - +// nEntryFlags - +// nTextureFlags - +// svBlockPath - +//----------------------------------------------------------------------------- +VPKEntryBlock_t::VPKEntryBlock_t(const vector &vData, int64_t nOffset, uint16_t nPreloadData, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, const string& svBlockPath) { m_nCrc32 = crc32::update(m_nCrc32, vData.data(), vData.size()); - m_nPreloadBytes = 0; + m_nPreloadData = nPreloadData; m_iArchiveIndex = nArchiveIndex; m_svBlockPath = svBlockPath; - int nEntryCount = (vData.size() + RVPK_MAX_BLOCK - 1) / RVPK_MAX_BLOCK; + int nEntryCount = (vData.size() + ENTRY_MAX - 1) / ENTRY_MAX; uint64_t nDataSize = vData.size(); int64_t nCurrentOffset = nOffset; for (int i = 0; i < nEntryCount; i++) { - uint64_t nSize = std::min(RVPK_MAX_BLOCK, nDataSize); + uint64_t nSize = std::min(ENTRY_MAX, nDataSize); nDataSize -= nSize; m_vvEntries.push_back(VPKEntryDescriptor_t(nEntryFlags, nTextureFlags, nCurrentOffset, nSize, nSize)); nCurrentOffset += nSize; } } +//----------------------------------------------------------------------------- +// Purpose: 'VPKEntryDescriptor_t' constructor +// Input : &nEntryFlags - +// &nTextureFlags - +// &nArchiveOffset - +// &nCompressedSize - +// &nUncompressedSize - +//----------------------------------------------------------------------------- VPKEntryDescriptor_t::VPKEntryDescriptor_t(uint32_t nEntryFlags, uint16_t nTextureFlags, uint64_t nArchiveOffset, uint64_t nCompressedSize, uint64_t nUncompressedSize) { m_nEntryFlags = nEntryFlags; @@ -395,30 +621,23 @@ VPKEntryDescriptor_t::VPKEntryDescriptor_t(CIOStream* pReader) //----------------------------------------------------------------------------- // Purpose: 'VPKDir_t' file constructor +// Input : &szPath - //----------------------------------------------------------------------------- VPKDir_t::VPKDir_t(const string& svPath) { CIOStream reader(svPath, CIOStream::Mode_t::READ); - reader.Read(this->m_vHeader.m_nFileMagic); + reader.Read(this->m_vHeader.m_nHeaderMarker); - if (this->m_vHeader.m_nFileMagic != RVPK_DIR_MAGIC) + if (this->m_vHeader.m_nHeaderMarker != VPK_HEADER_MARKER) { Error(eDLL_T::FS, "Error: vpk_dir file '%s' has invalid magic!\n", svPath.c_str()); - //return; + return; } - reader.Read(this->m_vHeader.m_nMajorVersion); // - reader.Read(this->m_vHeader.m_nMinorVersion); // - reader.Read(this->m_vHeader.m_nTreeSize); // - reader.Read(this->m_nFileDataSize); // - - DevMsg(eDLL_T::FS, "______________________________________________________________\n"); - DevMsg(eDLL_T::FS, "] HEADER_DETAILS ---------------------------------------------\n"); - DevMsg(eDLL_T::FS, "] File Magic : '%lu'\n", this->m_vHeader.m_nFileMagic); - DevMsg(eDLL_T::FS, "] Major Version : '%hu'\n", this->m_vHeader.m_nMajorVersion); - DevMsg(eDLL_T::FS, "] Minor Version : '%hu'\n", this->m_vHeader.m_nMinorVersion); - DevMsg(eDLL_T::FS, "] Tree Size : '%lu'\n", this->m_vHeader.m_nTreeSize); - DevMsg(eDLL_T::FS, "] File Data Size : '%lu'\n", this->m_nFileDataSize); + reader.Read(this->m_vHeader.m_nMajorVersion); // + reader.Read(this->m_vHeader.m_nMinorVersion); // + reader.Read(this->m_vHeader.m_nDirectorySize); // + reader.Read(this->m_nFileDataSize); // this->m_vvEntryBlocks = g_pPackedStore->GetEntryBlocks(&reader); this->m_svDirPath = svPath; // Set path to vpk_dir file. @@ -431,32 +650,28 @@ VPKDir_t::VPKDir_t(const string& svPath) } } - DevMsg(eDLL_T::FS, "______________________________________________________________\n"); - DevMsg(eDLL_T::FS, "] PACK_CHUNKS ------------------------------------------------\n"); - for (int i = 0; i < this->m_iArchiveCount + 1; i++) { string svArchivePath = g_pPackedStore->GetPackChunkFile(svPath, i); - DevMsg(eDLL_T::FS, "] '%s'\n", svArchivePath.c_str()); this->m_vsvArchives.push_back(svArchivePath); } } //----------------------------------------------------------------------------- -// Purpose: builds the VPKDir file -// Input : &svFileName - +// Purpose: builds the vpk directory file +// Input : &svDirectoryFile - // &vEntryBlocks - //----------------------------------------------------------------------------- -void VPKDir_t::Build(const string& svFileName, const vector& vEntryBlocks) +void VPKDir_t::Build(const string& svDirectoryFile, const vector& vEntryBlocks) { - CIOStream writer(svFileName, CIOStream::Mode_t::WRITE); + CIOStream writer(svDirectoryFile, CIOStream::Mode_t::WRITE); auto vMap = std::map>>(); - writer.Write(this->m_vHeader.m_nFileMagic); + writer.Write(this->m_vHeader.m_nHeaderMarker); writer.Write(this->m_vHeader.m_nMajorVersion); writer.Write(this->m_vHeader.m_nMinorVersion); - writer.Write(this->m_vHeader.m_nTreeSize); - writer.Write(this->m_vHeader.m_nTreeSize); + writer.Write(this->m_vHeader.m_nDirectorySize); + writer.Write(this->m_vHeader.m_nDirectorySize); for (VPKEntryBlock_t vBlock : vEntryBlocks) { @@ -465,7 +680,7 @@ void VPKDir_t::Build(const string& svFileName, const vector& vE if (svFilePath.empty()) { - svFilePath = " "; // Has to be padded with a space character if empty. + svFilePath = ' '; // Has to be padded with a space character if empty [root]. } if (!vMap.count(svExtension)) { @@ -489,7 +704,7 @@ void VPKDir_t::Build(const string& svFileName, const vector& vE writer.WriteString(GetFileName(eKeyValue.m_svBlockPath, true)); {/*Write entry block*/ writer.Write(eKeyValue.m_nCrc32); - writer.Write(eKeyValue.m_nPreloadBytes); + writer.Write(eKeyValue.m_nPreloadData); writer.Write(eKeyValue.m_iArchiveIndex); for (size_t i = 0; i < eKeyValue.m_vvEntries.size(); i++) @@ -520,10 +735,10 @@ void VPKDir_t::Build(const string& svFileName, const vector& vE writer.Write('\0'); } writer.Write('\0'); - m_vHeader.m_nTreeSize = static_cast(writer.GetPosition() - sizeof(VPKHeader_t)); + m_vHeader.m_nDirectorySize = static_cast(writer.GetPosition() - sizeof(VPKDirHeader_t)); - writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nTreeSize)); - writer.Write(this->m_vHeader.m_nTreeSize); + writer.SetPosition(offsetof(VPKDir_t, m_vHeader.m_nDirectorySize)); + writer.Write(this->m_vHeader.m_nDirectorySize); writer.Write(0); } /////////////////////////////////////////////////////////////////////////////// diff --git a/r5dev/vpklib/packedstore.h b/r5dev/vpklib/packedstore.h index 6842abdd..99da0a8c 100644 --- a/r5dev/vpklib/packedstore.h +++ b/r5dev/vpklib/packedstore.h @@ -2,25 +2,41 @@ #include "public/include/binstream.h" #include "thirdparty/lzham/include/lzham.h" -constexpr unsigned int RVPK_DIR_MAGIC = 'UŠ4'; -constexpr unsigned int RVPK_MAJOR_VER = 2; -constexpr unsigned int RVPK_MINOR_VER = 3; +constexpr unsigned int VPK_HEADER_MARKER = 0x55aa1234; +constexpr unsigned int VPK_MAJOR_VERSION = 2; +constexpr unsigned int VPK_MINOR_VERSION = 3; -constexpr unsigned int LIBRARY_PACKS = 2; -constexpr unsigned int LANGUAGE_PACKS = 11; constexpr unsigned int RVPK_DICT_SIZE = 20; -constexpr int RVPK_MAX_BLOCK = 1024 * 1024; -constexpr int DECOMP_MAX_BUF = 2024 * 2024; +constexpr int ENTRY_MAX = 1024 * 1024; +constexpr int COMP_MAX = 2024 * 2024; -const std::string DIR_LIBRARY_PREFIX[LIBRARY_PACKS] = { "server", "client" }; -const std::string DIR_LOCALE_PREFIX[LANGUAGE_PACKS] = { "english", "french", "german", "italian", "japanese", "korean", "polish", "portuguese", "russian", "spanish", "tchinese" }; +const vector DIR_CONTEXT = { "server", "client" }; +const vector DIR_LOCALE = { "english", "french", "german", "italian", "japanese", "korean", "polish", "portuguese", "russian", "spanish", "tchinese" }; + + +enum class EPackedEntryFlags : int +{ + ENTRY_NONE, + ENTRY_VISIBLE = 1 << 0, // FileSystem visibility? + ENTRY_CACHE = 1 << 8, // Only set for assets not stored in the depot directory. + ENTRY_TEXTURE_UNK0 = 1 << 18, + ENTRY_TEXTURE_UNK1 = 1 << 19, + ENTRY_TEXTURE_UNK2 = 1 << 20, +}; + +enum class EPackedTextureFlags : short +{ + TEXTURE_NONE, + TEXTURE_DEFAULT = 1 << 3, + TEXTURE_ENVIRONMENT_MAP = 1 << 10, +}; #pragma pack(push, 1) struct VPKFileEntry_t { - char* directory; - char* filename; - char* extension; + char* m_pszDirectory; + char* m_pszFileName; + char* m_pszExtension; uint8_t unknown[0x38]; }; @@ -53,37 +69,43 @@ struct VPKEntryDescriptor_t struct VPKEntryBlock_t { uint32_t m_nCrc32 {}; // Crc32 for the uncompressed block. - uint16_t m_nPreloadBytes{}; // Preload bytes. + uint16_t m_nPreloadData{}; // Preload bytes. uint16_t m_iArchiveIndex{}; // Index of the archive that contains this block. vector m_vvEntries {}; // Vector of all the entries of a given block (entries have a size limit of 1 MiB, so anything over is split into separate entries within the same block). string m_svBlockPath {}; // Path to block within vpk. - VPKEntryBlock_t(CIOStream* reader, string path); - VPKEntryBlock_t(const vector& vData, int64_t nOffset, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, string svBlockPath); + VPKEntryBlock_t(CIOStream* pReader, string svBlockPath); + VPKEntryBlock_t(const vector& vData, int64_t nOffset, uint16_t nPreloadData, uint16_t nArchiveIndex, uint32_t nEntryFlags, uint16_t nTextureFlags, const string& svBlockPath); }; -struct VPKHeader_t +struct VPKDirHeader_t { - uint32_t m_nFileMagic {}; // File magic. - uint16_t m_nMajorVersion{}; // Vpk major version. - uint16_t m_nMinorVersion{}; // Vpk minor version. - uint32_t m_nTreeSize {}; // Directory tree size. - uint32_t m_nPadding {}; // Padding to align the header. + uint32_t m_nHeaderMarker {}; // File magic. + uint16_t m_nMajorVersion {}; // Vpk major version. + uint16_t m_nMinorVersion {}; // Vpk minor version. + uint32_t m_nDirectorySize{}; // Directory tree size. + uint32_t m_nSignatureSize{}; // Directory signature. }; struct VPKDir_t { - VPKHeader_t m_vHeader {}; // Dir header. + VPKDirHeader_t m_vHeader {}; // Dir header. uint32_t m_nFileDataSize{}; // File data section size. vector m_vvEntryBlocks{}; // Vector of entry blocks. uint16_t m_iArchiveCount{}; // Highest archive index (archive count-1). vector m_vsvArchives {}; // Vector of archive file names. string m_svDirPath {}; // Path to vpk_dir file. - VPKDir_t() { m_vHeader.m_nFileMagic = RVPK_DIR_MAGIC; m_vHeader.m_nMajorVersion = RVPK_MAJOR_VER; m_vHeader.m_nMinorVersion = RVPK_MINOR_VER; }; + VPKDir_t() { m_vHeader.m_nHeaderMarker = VPK_HEADER_MARKER; m_vHeader.m_nMajorVersion = VPK_MAJOR_VERSION; m_vHeader.m_nMinorVersion = VPK_MINOR_VERSION; }; VPKDir_t(const string& svPath); - void Build(const string& svFileName, const vector& vEntryBlocks); + void Build(const string& svDirectoryFile, const vector& vEntryBlocks); +}; + +struct VPKPair_t +{ + string m_svBlockName; + string m_svDirectoryName; }; class CPackedStore @@ -101,14 +123,24 @@ class CPackedStore public: void InitLzCompParams(void); void InitLzDecompParams(void); - VPKDir_t GetPackDirFile(string svPackDirFile); - string GetPackChunkFile(const string& svPackDirFile, int iArchiveIndex); - vector GetEntryBlocks(CIOStream* reader); + + VPKDir_t GetPackDirFile(string svDirectoryFile) const; + string GetPackChunkFile(const string& svPackDirFile, int iArchiveIndex) const; + vector GetEntryBlocks(CIOStream* reader) const; vector GetEntryPaths(const string& svPathIn) const; - string FormatBlockPath(string svName, const string& svPath, const string& svExtension); - string StripLocalePrefix(const string& svPackDirFile); - void PackAll(const string& svDirIn, const string& svPathOut = ""); - void UnpackAll(VPKDir_t vpk, const string& svPathOut = ""); + vector GetEntryPaths(const string& svPathIn, const js::json& jManifest) const; + string GetLevelName(const string& svDirectoryName) const; + js::json GetManifest(const string& svWorkSpace, const string& svManifestName) const; + + string FormatBlockPath(string svName, const string& svPath, const string& svExtension) const; + string StripLocalePrefix(const string& svPackDirFile) const; + + VPKPair_t BuildFileName(string svLanguage, string svContext, const string& svPakName, int nPatch) const; + void BuildManifest(const vector& vBlock, const string& svWorkSpace, const string& svManifestName) const; + + void PackAll(const VPKPair_t& vPair, const string& svPathIn, const string& svPathOut, bool bManifestOnly); + void UnpackAll(const VPKDir_t& vpkDir, const string& svPathOut = ""); + void ValidateAdler32PostDecomp(const string& svDirAsset); void ValidateCRC32PostDecomp(const string& svDirAsset); }; diff --git a/r5dev/vstdlib/callback.cpp b/r5dev/vstdlib/callback.cpp index b36388ba..e1cdbce8 100644 --- a/r5dev/vstdlib/callback.cpp +++ b/r5dev/vstdlib/callback.cpp @@ -619,17 +619,17 @@ VPK_Pack_f */ void VPK_Pack_f(const CCommand& args) { - if (args.ArgC() < 2) + if (args.ArgC() < 4) { return; } - string szPathOut = "platform\\vpk"; - std::chrono::milliseconds msStart = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); + bool bManifestOnly = (args.ArgC() > 4); - //VPKDir_t vpk = g_pPackedStore->GetPackDirFile(args.Arg(1)); + std::chrono::milliseconds msStart = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); g_pPackedStore->InitLzCompParams(); - std::thread th([&] { g_pPackedStore->PackAll(args.Arg(1), szPathOut); }); + VPKPair_t vPair = g_pPackedStore->BuildFileName(args.Arg(1), args.Arg(2), args.Arg(3), NULL); + std::thread th([&] { g_pPackedStore->PackAll(vPair, fs_packedstore_workspace->GetString(), "vpk/", bManifestOnly); }); th.join(); std::chrono::milliseconds msEnd = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); @@ -650,7 +650,6 @@ void VPK_Unpack_f(const CCommand& args) { return; } - string szPathOut = "platform\\vpk"; std::chrono::milliseconds msStart = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); DevMsg(eDLL_T::FS, "______________________________________________________________\n"); @@ -660,7 +659,7 @@ void VPK_Unpack_f(const CCommand& args) VPKDir_t vpk = g_pPackedStore->GetPackDirFile(args.Arg(1)); g_pPackedStore->InitLzDecompParams(); - std::thread th([&] { g_pPackedStore->UnpackAll(vpk, szPathOut); }); + std::thread th([&] { g_pPackedStore->UnpackAll(vpk, ConvertToWinPath(fs_packedstore_workspace->GetString())); }); th.join(); std::chrono::milliseconds msEnd = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()); @@ -669,7 +668,7 @@ void VPK_Unpack_f(const CCommand& args) DevMsg(eDLL_T::FS, "______________________________________________________________\n"); DevMsg(eDLL_T::FS, "] OPERATION_DETAILS ------------------------------------------\n"); DevMsg(eDLL_T::FS, "] Time elapsed: '%.3f' seconds\n", (duration / 1000)); - DevMsg(eDLL_T::FS, "] Decompressed vpk to: '%s'\n", szPathOut.c_str()); + DevMsg(eDLL_T::FS, "] Decompressed vpk to: '%s'\n", fs_packedstore_workspace->GetString()); DevMsg(eDLL_T::FS, "--------------------------------------------------------------\n"); }