From 3f01894bfa063cc9f642906a74188f5a1bdb4787 Mon Sep 17 00:00:00 2001 From: Kawe Mazidjatari <48657826+Mauler125@users.noreply.github.com> Date: Tue, 16 Apr 2024 02:17:41 +0200 Subject: [PATCH] RTech: fixed streamed pak decoding progression bug During beta tests on a Linux system, we encountered an issue where the I/O could be slower and thus result in not enough bytes being streamed by the time we invoke the decoder (lenStreamed == bufSizeNeeded at this point), after 16 calls with lenStreamed == bufSizeNeeded, the engine errors as ZSTD_NO_FORWARD_PROGRESS_MAX would be reached. Added value of 'ZSTD_nextSrcSizeToDecompress()' to 'PakDecoder_s::bufferSizeNeeded' to make sure we never call the decoder without any new streamed bytes. Also increased the value of ZSTD_NO_FORWARD_PROGRESS_MAX to 1024 since this fixed the issue without applying the aforementioned patch, this was increased as a hardening measure. --- src/rtech/pak/pakdecode.cpp | 7 ++++--- src/thirdparty/zstd/CMakeLists.txt | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rtech/pak/pakdecode.cpp b/src/rtech/pak/pakdecode.cpp index 7037c264..c9075521 100644 --- a/src/rtech/pak/pakdecode.cpp +++ b/src/rtech/pak/pakdecode.cpp @@ -620,7 +620,8 @@ bool Pak_ZStdStreamDecode(PakDecoder_s* const decoder, const PakRingBufferFrame_ inFrame.frameLen, NULL }; - const size_t ret = ZSTD_decompressStream(decoder->zstreamContext, &outBuffer, &inBuffer); + ZSTD_DStream* const dctx = decoder->zstreamContext; + const size_t ret = ZSTD_decompressStream(dctx, &outBuffer, &inBuffer); if (ZSTD_isError(ret)) { @@ -646,14 +647,14 @@ bool Pak_ZStdStreamDecode(PakDecoder_s* const decoder, const PakRingBufferFrame_ // // if the input stream has fully decoded, this should equal the size of the // encoded pak file - decoder->bufferSizeNeeded = decoder->inBufBytePos; + decoder->bufferSizeNeeded = decoder->inBufBytePos + ZSTD_nextSrcSizeToDecompress(dctx); const bool decoded = ret == NULL; // zstd decoder no longer necessary at this point, deallocate if (decoded) { - ZSTD_freeDStream(decoder->zstreamContext); + ZSTD_freeDStream(dctx); decoder->zstreamContext = nullptr; } diff --git a/src/thirdparty/zstd/CMakeLists.txt b/src/thirdparty/zstd/CMakeLists.txt index 9f500816..36e490ee 100644 --- a/src/thirdparty/zstd/CMakeLists.txt +++ b/src/thirdparty/zstd/CMakeLists.txt @@ -94,5 +94,6 @@ thirdparty_suppress_warnings() target_compile_definitions( ${PROJECT_NAME} PRIVATE "ZSTD_MULTITHREAD" + "ZSTD_NO_FORWARD_PROGRESS_MAX=1024" "DEBUGLEVEL=0" )