ReVPK: bug fixes & improvements

* Make sure the workspace path actually exists before attempting to pack it.
* Make sure the VPK directory tree file was parsed correctly before unpacking store.
* Log debug output for each pack operation to a file.
* Fix bug in s_DirFileRegex regex pattern, which would include a trailing '_' in the context part of the directory tree file name.
* Fix bug in 'GetLevelName()' and 'GetDirNameParts()' causing it to parse the path as well, prune the path before running the regex.
* Renamed 'GetLevelName()' to 'PackedStore_GetDirLevelName()'.
* Renamed 'GetDirNameParts()' to 'PackedStore_GetDirNameParts()'.
* Write a front-end enable file when building client VPK's.

r5dev/core/logdef.cpp

@@ -4,6 +4,8 @@
 std::shared_ptr<spdlog::logger> g_TermLogger;
 std::shared_ptr<spdlog::logger> g_ImGuiLogger;
 
+std::shared_ptr<spdlog::logger> g_SuppementalToolsLogger;
+
 std::ostringstream g_LogStream;
 std::shared_ptr<spdlog::sinks::ostream_sink_st> g_LogSink;
 
@@ -66,6 +68,14 @@ void SpdLog_Init(const bool bAnsiColor)
 	bInitialized = true;
 }
 
+//#############################################################################
+// SPDLOG SHUTDOWN
+//#############################################################################
+void SpdLog_Shutdown()
+{
+	spdlog::shutdown();
+}
+
 void SpdLog_Create()
 {
 	/************************
@@ -91,10 +101,11 @@ void SpdLog_Create()
 		, fmt::format("{:s}\\{:s}", g_LogSessionDirectory, "filesystem.log"), SPDLOG_MAX_SIZE, SPDLOG_NUM_FILE)->set_pattern("[%Y-%m-%d %H:%M:%S.%e] %v");
 }
 
-//#############################################################################
-// SPDLOG SHUTDOWN
-//#############################################################################
-void SpdLog_Shutdown()
+#ifdef _TOOLS
+// NOTE: used for tools as additional file logger on top of the existing terminal logger.
+void SpdLog_InstallSupplementalLogger(const char* pszLoggerName, const char* pszLogFileName, const char* pszPattern)
 {
-	spdlog::shutdown();
+	g_SuppementalToolsLogger = spdlog::basic_logger_mt(pszLoggerName, pszLogFileName);
+	g_SuppementalToolsLogger->set_pattern(pszPattern);
 }
+#endif // _TOOLS

r5dev/core/logdef.h

@@ -18,11 +18,18 @@ inline bool g_bSpdLog_UseAnsiClr = false;
 extern std::shared_ptr<spdlog::logger> g_TermLogger;
 extern std::shared_ptr<spdlog::logger> g_ImGuiLogger;
 
+#ifdef _TOOLS
+extern std::shared_ptr<spdlog::logger> g_SuppementalToolsLogger;
+#endif // _TOOLS
+
 //-------------------------------------------------------------------------
 // IMGUI CONSOLE SINK                                                     |
 extern std::ostringstream g_LogStream;
 extern std::shared_ptr<spdlog::sinks::ostream_sink_st> g_LogSink;
 
 void SpdLog_Init(const bool bAnsiColor);
-void SpdLog_Create(void);
 void SpdLog_Shutdown(void);
+
+#ifdef _TOOLS
+void SpdLog_InstallSupplementalLogger(const char* pszLoggerName, const char* pszLogFileName, const char* pszPattern = "[%Y-%m-%d %H:%M:%S.%e] %v");
+#endif // _TOOLS

r5dev/core/logger.cpp

@@ -307,7 +307,12 @@ void EngineLoggerSink(LogType_t logType, LogLevel_t logLevel, eDLL_T context,
 	g_LogStream.clear();
 #endif // !DEDICATED
 
-#endif // !_TOOLS
+#else
+	if (g_SuppementalToolsLogger)
+	{
+		g_SuppementalToolsLogger->debug(message);
+	}
+#endif
 
 	if (exitCode) // Terminate the process if an exit code was passed.
 	{

r5dev/revpk/revpk.cpp

@@ -19,8 +19,14 @@
 #define PACK_COMMAND "pack"
 #define UNPACK_COMMAND "unpack"
 
+#define PACK_LOG_DIR "manifest/pack_logs/"
+#define UNPACK_LOG_DIR "manifest/unpack_logs/"
+
+#define FRONTEND_ENABLE_FILE "enable.txt"
+
 static CKeyValuesSystem s_KeyValuesSystem;
 static CFileSystem_Stdio g_FullFileSystem;
+static bool s_bUseAnsiColors = true;
 
 //-----------------------------------------------------------------------------
 // Purpose: keyvalues singleton accessor
@@ -48,8 +54,8 @@ static void ReVPK_Init()
     g_CoreMsgVCallback = EngineLoggerSink;
     lzham_enable_fail_exceptions(true);
 
-    Console_Init(true);
-    SpdLog_Init(true);
+    Console_Init(s_bUseAnsiColors);
+    SpdLog_Init(s_bUseAnsiColors);
 }
 
 //-----------------------------------------------------------------------------
@@ -101,6 +107,35 @@ static void ReVPK_Usage()
     Warning(eDLL_T::FS, "%s", usage.Get());
 }
 
+//-----------------------------------------------------------------------------
+// Purpose: writes the VPK front-end enable file
+//-----------------------------------------------------------------------------
+static void ReVPK_WriteEnableFile(const char* containingPath)
+{
+    CFmtStr1024 textFileName;
+    textFileName.Format("%s%s", containingPath, FRONTEND_ENABLE_FILE);
+
+    if (FileSystem()->FileExists(textFileName.String(), "PLATFORM"))
+    {
+        // Already exists.
+        return;
+    }
+
+    FileHandle_t enableTxt = FileSystem()->Open(textFileName.String(), "wb", "PLATFORM");
+
+    if (enableTxt)
+    {
+        const char* textData = "1 \r\n";
+        FileSystem()->Write(textData, strlen(textData), enableTxt);
+        FileSystem()->Close(enableTxt);
+    }
+    else
+    {
+        Error(eDLL_T::FS, NO_ERROR, "Failed to write front-end enable file \"%s\"; insufficient rights?\n",
+            textFileName.String());
+    }
+}
+
 //-----------------------------------------------------------------------------
 // Purpose: packs VPK files into 'SHIP' VPK directory
 //-----------------------------------------------------------------------------
@@ -114,10 +149,38 @@ static void ReVPK_Pack(const CCommand& args)
         return;
     }
 
-    VPKPair_t pair(args.Arg(2), args.Arg(3), args.Arg(4), NULL);
-    CFastTimer timer;
+    CFmtStr1024 workspacePath = argCount > 5 ? args.Arg(5) : "ship/";
+    CFmtStr1024 buildPath     = argCount > 6 ? args.Arg(6) : "vpk/";
 
+    // Make sure there's always a trailing slash.
+    V_AppendSlash(workspacePath.Access(), workspacePath.GetMaxLength(), '/');
+    V_AppendSlash(buildPath.Access(), buildPath.GetMaxLength(), '/');
+
+    if (!FileSystem()->IsDirectory(workspacePath, "PLATFORM"))
+    {
+        Error(eDLL_T::FS, NO_ERROR, "Workspace path \"%s\" doesn't exist!\n", workspacePath.String());
+        return;
+    }
+
+    const char* localeName = args.Arg(2);
+    const char* contextName = args.Arg(3);
+    const char* levelName = args.Arg(4);
+
+    // For clients, we need an enable file which the engine uses to determine
+    // whether or not to mount the front-end VPK file.
+    if (V_strcmp(contextName, DIR_TARGET[EPackedStoreTargets::STORE_TARGET_CLIENT]) == NULL)
+    {
+        ReVPK_WriteEnableFile(buildPath);
+    }
+
+    // Write the pack log to a file.
+    CFmtStr1024 textFileName("%s%s%s%s_%s.log", buildPath.String(), PACK_LOG_DIR, localeName, contextName, levelName);
+    SpdLog_InstallSupplementalLogger("supplemental_logger_mt", textFileName.String());
+
+    VPKPair_t pair(localeName, contextName, levelName, NULL);
     Msg(eDLL_T::FS, "*** Starting VPK build command for: '%s'\n", pair.m_DirName.Get());
+
+    CFastTimer timer;
     timer.Start();
 
     CPackedStoreBuilder builder;
@@ -126,9 +189,7 @@ static void ReVPK_Pack(const CCommand& args)
         argCount > 7 ? (std::min)(atoi(args.Arg(7)), LZHAM_MAX_HELPER_THREADS) : -1, // Num threads.
         argCount > 8 ? args.Arg(8) : "default"); // Compress level.
 
-    builder.PackStore(pair,
-        argCount > 5 ? args.Arg(5) : "ship/", // Workspace path.
-        argCount > 6 ? args.Arg(6) : "vpk/"); // build path.
+    builder.PackStore(pair, workspacePath.String(), buildPath.String());
 
     timer.End();
     Msg(eDLL_T::FS, "*** Time elapsed: '%lf' seconds\n", timer.GetDuration().GetSeconds());
@@ -148,12 +209,30 @@ static void ReVPK_Unpack(const CCommand& args)
         return;
     }
 
-    CUtlString arg = args.Arg(2);
+    const char* fileName = args.Arg(2);
+    const char* outPath = argCount > 3 ? args.Arg(3) : "ship/";
+    const bool sanitize = argCount > 4 ? atoi(args.Arg(4)) != NULL : false;
+
+    VPKDir_t vpk(fileName, sanitize);
+
+    // Make sure the VPK directory tree was actually parsed correctly.
+    if (vpk.Failed())
+    {
+        Error(eDLL_T::FS, NO_ERROR, "Failed to parse directory tree file \"%s\"!\n", fileName);
+        return;
+    }
+
+    CUtlString prefixName = PackedStore_GetDirNameParts(vpk.m_DirFilePath, 1);
+    CUtlString levelName = PackedStore_GetDirNameParts(vpk.m_DirFilePath, 2);
+
+    // Write the unpack log to a file.
+    CFmtStr1024 textFileName("%s%s%s_%s.log", outPath, UNPACK_LOG_DIR, prefixName.String(), levelName.String());
+    SpdLog_InstallSupplementalLogger("supplemental_logger_mt", textFileName.String());
+
+    const char* actualDirFile = vpk.m_DirFilePath.String();
+    Msg(eDLL_T::FS, "*** Starting VPK extraction command for: '%s'\n", actualDirFile);
 
-    VPKDir_t vpk(arg, (argCount > 4));
     CFastTimer timer;
-
-    Msg(eDLL_T::FS, "*** Starting VPK extraction command for: '%s'\n", arg.Get());
     timer.Start();
 
     CPackedStoreBuilder builder;
@@ -184,17 +263,21 @@ int main(int argc, char* argv[])
 
     args.Tokenize(str.Get(), cmd_source_t::kCommandSrcCode);
 
-    if (!args.ArgC())
+    if (!args.ArgC()) {
         ReVPK_Usage();
+    }
     else
     {
-        if (strcmp(args.Arg(1), PACK_COMMAND) == NULL)
+        if (V_strcmp(args.Arg(1), PACK_COMMAND) == NULL) {
             ReVPK_Pack(args);
-        else if (strcmp(args.Arg(1), UNPACK_COMMAND) == NULL)
+        }
+        else if (V_strcmp(args.Arg(1), UNPACK_COMMAND) == NULL) {
             ReVPK_Unpack(args);
-        else
+        }
+        else {
             ReVPK_Usage();
         }
+    }
 
     ReVPK_Shutdown();
 }

r5dev/vpklib/packedstore.cpp

@@ -35,7 +35,7 @@
 
 extern CFileSystem_Stdio* FileSystem();
 
-static const std::regex s_DirFileRegex{ R"((?:.*\/)?([^_]*_)(.*)(.bsp.pak000_dir).*)" };
+static const std::regex s_DirFileRegex{ R"((?:.*\/)?([^_]*)(?:_)(.*)(.bsp.pak000_dir).*)" };
 static const std::regex s_BlockFileRegex{ R"(pak000_([0-9]{3}))" };
 
 //-----------------------------------------------------------------------------
@@ -88,6 +88,40 @@ void CPackedStoreBuilder::InitLzDecoder(void)
 	m_Decoder.m_pSeed_bytes      = NULL;
 }
 
+//-----------------------------------------------------------------------------
+// Purpose: gets the level name from the directory file name
+// Input  : &dirFileName - 
+// Output : level name as string (e.g. "mp_rr_box")
+//-----------------------------------------------------------------------------
+CUtlString PackedStore_GetDirLevelName(const CUtlString& dirFileName)
+{
+	const char* baseFileName = V_UnqualifiedFileName(dirFileName.String());
+
+	std::cmatch regexMatches;
+	std::regex_search(baseFileName, regexMatches, s_DirFileRegex);
+
+	CUtlString result;
+	result.Format("%s%s", regexMatches[1].str().c_str(), regexMatches[2].str().c_str());
+
+	return result;
+}
+
+//-----------------------------------------------------------------------------
+// Purpose: gets the parts of the directory file name
+// Input  : &dirFileName   - 
+//          nCaptureGroup  - (1 = locale + target, 2 = level)
+// Output : part of directory file name as string
+//-----------------------------------------------------------------------------
+CUtlString PackedStore_GetDirNameParts(const CUtlString& dirFileName, const int nCaptureGroup)
+{
+	const char* baseFileName = V_UnqualifiedFileName(dirFileName.String());
+
+	std::cmatch regexMatches;
+	std::regex_search(baseFileName, regexMatches, s_DirFileRegex);
+
+	return regexMatches[nCaptureGroup].str().c_str();
+}
+
 //-----------------------------------------------------------------------------
 // Purpose: formats the file entry path
 // Input  : &filePath - 
@@ -161,22 +195,6 @@ static bool ShouldPrune(const CUtlString& filePath, CUtlVector<CUtlString>& igno
 	return false;
 }
 
-//-----------------------------------------------------------------------------
-// Purpose: gets the level name from the directory file name
-// Input  : &dirFileName - 
-// Output : level name as string (e.g. "mp_rr_box")
-//-----------------------------------------------------------------------------
-static CUtlString GetLevelName(const CUtlString& dirFileName)
-{
-	std::cmatch regexMatches;
-	std::regex_search(dirFileName.Get(), regexMatches, s_DirFileRegex);
-
-	CUtlString result;
-	result.Format("%s%s", regexMatches[1].str().c_str(), regexMatches[2].str().c_str());
-
-	return result;
-}
-
 //-----------------------------------------------------------------------------
 // Purpose: gets the manifest file associated with the VPK name (must be freed after wards)
 // Input  : &workspacePath - 
@@ -278,7 +296,7 @@ static void GetEntryBlocks(CUtlVector<VPKEntryBlock_t>& entryBlocks, FileHandle_
 static bool GetEntryValues(CUtlVector<VPKKeyValues_t>& entryValues, 
 	const CUtlString& workspacePath, const CUtlString& dirFileName)
 {
-	KeyValues* pManifestKV = GetManifest(workspacePath, GetLevelName(dirFileName));
+	KeyValues* pManifestKV = GetManifest(workspacePath, PackedStore_GetDirLevelName(dirFileName));
 
 	if (!pManifestKV)
 	{
@@ -578,7 +596,7 @@ void CPackedStoreBuilder::UnpackStore(const VPKDir_t& vpkDir, const char* worksp
 		return;
 	}
 
-	BuildManifest(vpkDir.m_EntryBlocks, workspacePath, GetLevelName(vpkDir.m_DirFilePath));
+	BuildManifest(vpkDir.m_EntryBlocks, workspacePath, PackedStore_GetDirLevelName(vpkDir.m_DirFilePath));
 	const CUtlString basePath = vpkDir.m_DirFilePath.StripFilename(false);
 
 	for (uint16_t packFileIndex : vpkDir.m_PakFileIndices)
@@ -820,20 +838,6 @@ VPKPair_t::VPKPair_t(const char* pLocale, const char* pTarget, const char* pLeve
 	m_DirName.Format("%s%s_%s.bsp.pak000_dir.vpk", pLocale, pTarget, pLevel);
 }
 
-//-----------------------------------------------------------------------------
-// Purpose: gets the parts of the directory file name
-// Input  : &dirFileName   - 
-//          nCaptureGroup  - (1 = locale + target, 2 = level)
-// Output : part of directory file name as string
-//-----------------------------------------------------------------------------
-CUtlString VPKPair_t::GetNameParts(const int nCaptureGroup)
-{
-	std::cmatch regexMatches;
-	std::regex_search(m_DirName.Get(), regexMatches, s_DirFileRegex);
-
-	return regexMatches[nCaptureGroup].str().c_str();
-}
-
 //-----------------------------------------------------------------------------
 // Purpose: 'VPKDir_t' file constructor
 // Input  : &dirFilePath - 

r5dev/vpklib/packedstore.h

@@ -192,7 +192,6 @@ struct VPKPair_t
 	CUtlString m_DirName;
 
 	VPKPair_t(const char* svLocale, const char* svTarget, const char* svLevel, int nPatch);
-	CUtlString GetNameParts(const int nCaptureGroup);
 };
 
 //-----------------------------------------------------------------------------
@@ -214,6 +213,9 @@ private:
 	lzham_decompress_params m_Decoder; // LZham decompression parameters.
 	std::unordered_map<string, const VPKChunkDescriptor_t&> m_ChunkHashMap;
 };
+
+CUtlString PackedStore_GetDirLevelName(const CUtlString& dirFileName);
+CUtlString PackedStore_GetDirNameParts(const CUtlString& dirFileName, const int nCaptureGroup);
 ///////////////////////////////////////////////////////////////////////////////
 
 #endif // PACKEDSTORE_H