R5Reloaded/r5sdk
1
0
Fork 0
mirror of https://github.com/Mauler125/r5sdk.git synced 2025-02-09 19:15:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Remove experimental CRC implementation in LZHAM

Browse Source 
This implementation was done to calculate buffer CRC on the go, as that was what the VPK file system uses for the file crc's. But it later turned out to be unnecessary, and was never reverted.
This commit is contained in:
Kawe Mazidjatari 2022-12-04 01:33:10 +01:00
parent 10a0a41162
commit 55b90781ee
12 changed files with 65 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
r5dev/public/utility/sigcache.cpp
View File

@ -81,17 +81,17 @@ bool CSigCache::LoadCache(const string& svCacheFile)
header.m_nBlobSizeMem = reader.Read<uint64_t>();
header.m_nBlobSizeDisk = reader.Read<uint64_t>();
header.m_nBlobHash = reader.Read<uint32_t>();
header.m_nBlobChecksum = reader.Read<uint32_t>();
uint32_t nCrc32;
uint32_t nAdler32;
std::unique_ptr<uint8_t[]> pSrcBuf(new uint8_t[header.m_nBlobSizeDisk]);
std::unique_ptr<uint8_t[]> pDstBuf(new uint8_t[header.m_nBlobSizeMem]);
reader.Read<uint8_t>(*pSrcBuf.get(), header.m_nBlobSizeDisk);
DecompressBlob(header.m_nBlobSizeDisk, header.m_nBlobSizeMem, nCrc32, pSrcBuf.get(), pDstBuf.get());
DecompressBlob(header.m_nBlobSizeDisk, header.m_nBlobSizeMem, nAdler32, pSrcBuf.get(), pDstBuf.get());
if (header.m_nBlobHash != nCrc32)
if (header.m_nBlobChecksum != nAdler32)
{
return false;
}
@ -135,7 +135,7 @@ bool CSigCache::WriteCache(const string& svCacheFile)
header.m_nBlobSizeMem = svBuffer.size();
uint64_t nCompSize = svBuffer.size();
if (!CompressBlob(svBuffer.size(), nCompSize, header.m_nBlobHash, reinterpret_cast<const uint8_t*>(svBuffer.data()), pBuffer.get()))
if (!CompressBlob(svBuffer.size(), nCompSize, header.m_nBlobChecksum, reinterpret_cast<const uint8_t*>(svBuffer.data()), pBuffer.get()))
{
return false;
}
@ -152,19 +152,19 @@ bool CSigCache::WriteCache(const string& svCacheFile)
// Purpose: decompresses the blob containing the signature map
// Input : nSrcLen -
// &nDstLen -
// &nCrc32 -
// &nAdler -
// *pSrcBuf -
// *pDstBuf -
// Output : true on success, false otherwise
//-----------------------------------------------------------------------------
bool CSigCache::DecompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const
bool CSigCache::DecompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nAdler, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const
{
lzham_decompress_params lzDecompParams{};
lzDecompParams.m_dict_size_log2 = SIGDB_DICT_SIZE;
lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED | lzham_decompress_flags::LZHAM_DECOMP_FLAG_COMPUTE_CRC32;
lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED | lzham_decompress_flags::LZHAM_DECOMP_FLAG_COMPUTE_ADLER32;
lzDecompParams.m_struct_size = sizeof(lzham_decompress_params);
lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&lzDecompParams, pDstBuf, &nDstLen, pSrcBuf, nSrcLen, NULL, &nCrc32);
lzham_decompress_status_t lzDecompStatus = lzham_decompress_memory(&lzDecompParams, pDstBuf, &nDstLen, pSrcBuf, nSrcLen, &nAdler);
if (lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
{
@ -179,19 +179,19 @@ bool CSigCache::DecompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nCrc32
// Purpose: compresses the blob containing the signature map
// Input : nSrcLen -
// &nDstLen -
// &nCrc32 -
// &nAdler -
// *pSrcBuf -
// *pDstBuf -
// Output : true on success, false otherwise
//-----------------------------------------------------------------------------
bool CSigCache::CompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const
bool CSigCache::CompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nAdler, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const
{
lzham_compress_params lzCompParams{};
lzCompParams.m_dict_size_log2 = SIGDB_DICT_SIZE;
lzCompParams.m_level = lzham_compress_level::LZHAM_COMP_LEVEL_FASTEST;
lzCompParams.m_compress_flags = lzham_compress_flags::LZHAM_COMP_FLAG_DETERMINISTIC_PARSING;
lzham_compress_status_t lzCompStatus = lzham_compress_memory(&lzCompParams, pDstBuf, &nDstLen, pSrcBuf, nSrcLen, NULL, &nCrc32);
lzham_compress_status_t lzCompStatus = lzham_compress_memory(&lzCompParams, pDstBuf, &nDstLen, pSrcBuf, nSrcLen, &nAdler);
if (lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
{

6
r5dev/public/utility/sigcache.h
View File

@ -28,8 +28,8 @@ public:
bool WriteCache(const string& svCacheFile);
private:
bool CompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const;
bool DecompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nCrc32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const;
bool CompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nAdler32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const;
bool DecompressBlob(size_t nSrcLen, size_t& nDstLen, uint32_t& nAdler32, const uint8_t* pSrcBuf, uint8_t* pDstBuf) const;
SigMap_Pb m_Cache;
bool m_bInitialized;
@ -43,7 +43,7 @@ struct SigDBHeader_t
uint16_t m_nMinorVersion;
uint64_t m_nBlobSizeMem;
uint64_t m_nBlobSizeDisk;
uint32_t m_nBlobHash;
uint32_t m_nBlobChecksum;
};
#pragma pack(pop)

43
r5dev/thirdparty/lzham/include/lzham.h vendored
View File

@ -146,12 +146,6 @@ extern "C" {
const void *m_pSeed_bytes; // for delta compression (optional) - pointer to seed bytes buffer, must be at least m_num_seed_bytes long
} lzham_compress_params;
typedef struct
{
lzham_uint32 adler32;
lzham_uint32 crc32;
} lzham_compress_checksums;
// Initializes a compressor. Returns a pointer to the compressor's internal state, or NULL on failure.
// pParams cannot be NULL. Be sure to initialize the pParams->m_struct_size member to sizeof(lzham_compress_params) (along with the other members to reasonable values) before calling this function.
// TODO: With large dictionaries this function could take a while (due to memory allocation). I need to add a reinit() API for compression (decompression already has one).
@ -160,8 +154,8 @@ extern "C" {
LZHAM_DLL_EXPORT lzham_compress_state_ptr LZHAM_CDECL lzham_compress_reinit(lzham_compress_state_ptr pState);
// Deinitializes a compressor, releasing all allocated memory.
// returns adler32 and crc32 of source data (valid only on success).
LZHAM_DLL_EXPORT lzham_compress_checksums LZHAM_CDECL lzham_compress_deinit(lzham_compress_state_ptr pState);
// returns adler32 of source data (valid only on success).
LZHAM_DLL_EXPORT lzham_uint32 LZHAM_CDECL lzham_compress_deinit(lzham_compress_state_ptr pState);
// Compresses an arbitrarily sized block of data, writing as much available compressed data as possible to the output buffer.
// This method may be called as many times as needed, but for best perf. try not to call it with tiny buffers.
@ -198,8 +192,7 @@ extern "C" {
size_t *pDst_len,
const lzham_uint8* pSrc_buf,
size_t src_len,
lzham_uint32 *pAdler32,
lzham_uint32 *pCrc32);
lzham_uint32 *pAdler32);
// Decompression
typedef enum
@ -230,7 +223,6 @@ extern "C" {
LZHAM_DECOMP_STATUS_FAILED_EXPECTED_MORE_RAW_BYTES,
LZHAM_DECOMP_STATUS_FAILED_BAD_CODE,
LZHAM_DECOMP_STATUS_FAILED_ADLER32,
LZHAM_DECOMP_STATUS_FAILED_CRC32,
LZHAM_DECOMP_STATUS_FAILED_BAD_RAW_BLOCK,
LZHAM_DECOMP_STATUS_FAILED_BAD_COMP_BLOCK_SYNC_CHECK,
LZHAM_DECOMP_STATUS_FAILED_BAD_ZLIB_HEADER,
@ -247,7 +239,6 @@ extern "C" {
LZHAM_DECOMP_FLAG_NONE = 0,
LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED = 1 << 0,
LZHAM_DECOMP_FLAG_COMPUTE_ADLER32 = 1 << 1,
LZHAM_DECOMP_FLAG_COMPUTE_CRC32 = 1 << 2,
LZHAM_DECOMP_FLAG_READ_ZLIB_STREAM = 1 << 3,
} lzham_decompress_flags;
@ -265,12 +256,6 @@ extern "C" {
const void *m_pSeed_bytes; // for delta compression (optional) - pointer to seed bytes buffer, must be at least m_num_seed_bytes long
} lzham_decompress_params;
typedef struct
{
lzham_uint32 adler32;
lzham_uint32 crc32;
} lzham_decompress_checksums;
// Initializes a decompressor.
// pParams cannot be NULL. Be sure to initialize the pParams->m_struct_size member to sizeof(lzham_decompress_params) (along with the other members to reasonable values) before calling this function.
// Note: With large dictionaries this function could take a while (due to memory allocation). To serially decompress multiple streams, it's faster to init a compressor once and
@ -282,7 +267,7 @@ extern "C" {
// Deinitializes a decompressor.
// returns adler32 of decompressed data if compute_adler32 was true, otherwise it returns the adler32 from the compressed stream.
LZHAM_DLL_EXPORT lzham_decompress_checksums LZHAM_CDECL lzham_decompress_deinit(lzham_decompress_state_ptr pState);
LZHAM_DLL_EXPORT lzham_uint32 LZHAM_CDECL lzham_decompress_deinit(lzham_decompress_state_ptr pState);
// Decompresses an arbitrarily sized block of compressed data, writing as much available decompressed data as possible to the output buffer.
// This method is implemented as a coroutine so it may be called as many times as needed. However, for best perf. try not to call it with tiny buffers.
@ -311,8 +296,7 @@ extern "C" {
size_t *pDst_len,
const lzham_uint8* pSrc_buf,
size_t src_len,
lzham_uint32 *pAdler32,
lzham_uint32 *pCrc32);
lzham_uint32 *pAdler32);
// ------------------- zlib-style API Definitions.
@ -328,11 +312,11 @@ extern "C" {
#define LZHAM_Z_ADLER32_INIT (1)
// lzham_adler32() returns the initial adler-32 value to use when called with ptr==NULL.
LZHAM_DLL_EXPORT lzham_z_ulong lzham_z_adler32(lzham_z_ulong adler, const unsigned char *ptr, size_t buf_len);
LZHAM_DLL_EXPORT lzham_z_ulong lzham_z_adler32(lzham_z_ulong adler, const lzham_uint8 *ptr, size_t buf_len);
#define LZHAM_Z_CRC32_INIT (0)
// lzham_crc32() returns the initial CRC-32 value to use when called with ptr==NULL.
LZHAM_DLL_EXPORT lzham_z_ulong lzham_z_crc32(lzham_z_ulong crc, const unsigned char *ptr, size_t buf_len);
// lzham_crc32() returns the initial crc-32 value to use when called with ptr==NULL.
LZHAM_DLL_EXPORT lzham_z_ulong lzham_z_crc32(lzham_z_ulong crc, const lzham_uint8 *ptr, size_t buf_len);
// Compression strategies.
enum
@ -437,8 +421,7 @@ extern "C" {
void *opaque; // heap alloc function user pointer
int data_type; // data_type (unused)
lzham_z_ulong adler32; // adler32 of the source or uncompressed data
lzham_z_ulong crc32; // crc32 of the source or uncompressed data
lzham_z_ulong adler32; // adler32 of the source or uncompressed data
lzham_z_ulong reserved; // not used
} lzham_z_stream;
@ -618,16 +601,16 @@ extern "C" {
typedef lzham_compress_state_ptr (LZHAM_CDECL *lzham_compress_init_func)(const lzham_compress_params *pParams);
typedef lzham_compress_state_ptr (LZHAM_CDECL *lzham_compress_reinit_func)(lzham_compress_state_ptr pState);
typedef lzham_compress_checksums (LZHAM_CDECL *lzham_compress_deinit_func)(lzham_compress_state_ptr pState);
typedef lzham_uint32 (LZHAM_CDECL *lzham_compress_deinit_func)(lzham_compress_state_ptr pState);
typedef lzham_compress_status_t (LZHAM_CDECL *lzham_compress_func)(lzham_compress_state_ptr pState, const lzham_uint8 *pIn_buf, size_t *pIn_buf_size, lzham_uint8 *pOut_buf, size_t *pOut_buf_size, lzham_bool no_more_input_bytes_flag);
typedef lzham_compress_status_t (LZHAM_CDECL *lzham_compress2_func)(lzham_compress_state_ptr pState, const lzham_uint8 *pIn_buf, size_t *pIn_buf_size, lzham_uint8 *pOut_buf, size_t *pOut_buf_size, lzham_flush_t flush_type);
typedef lzham_compress_status_t (LZHAM_CDECL *lzham_compress_memory_func)(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32* pCrc32);
typedef lzham_compress_status_t (LZHAM_CDECL *lzham_compress_memory_func)(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32);
typedef lzham_decompress_state_ptr (LZHAM_CDECL *lzham_decompress_init_func)(const lzham_decompress_params *pParams);
typedef lzham_decompress_state_ptr (LZHAM_CDECL *lzham_decompress_reinit_func)(lzham_compress_state_ptr pState, const lzham_decompress_params *pParams);
typedef lzham_decompress_checksums (LZHAM_CDECL *lzham_decompress_deinit_func)(lzham_decompress_state_ptr pState);
typedef lzham_uint32 (LZHAM_CDECL *lzham_decompress_deinit_func)(lzham_decompress_state_ptr pState);
typedef lzham_decompress_status_t (LZHAM_CDECL *lzham_decompress_func)(lzham_decompress_state_ptr pState, const lzham_uint8 *pIn_buf, size_t *pIn_buf_size, lzham_uint8 *pOut_buf, size_t *pOut_buf_size, lzham_bool no_more_input_bytes_flag);
typedef lzham_decompress_status_t (LZHAM_CDECL *lzham_decompress_memory_func)(const lzham_decompress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32 *pCrc32);
typedef lzham_decompress_status_t (LZHAM_CDECL *lzham_decompress_memory_func)(const lzham_decompress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32);
typedef const char *(LZHAM_CDECL *lzham_z_version_func)(void);
typedef int (LZHAM_CDECL *lzham_z_deflateInit_func)(lzham_z_streamp pStream, int level);

14
r5dev/thirdparty/lzham/lzham_api.cpp vendored
View File

@ -24,7 +24,7 @@ extern "C" LZHAM_DLL_EXPORT lzham_decompress_state_ptr lzham_decompress_reinit(l
return lzham::lzham_lib_decompress_reinit(p, pParams);
}
extern "C" LZHAM_DLL_EXPORT lzham_decompress_checksums lzham_decompress_deinit(lzham_decompress_state_ptr p)
extern "C" LZHAM_DLL_EXPORT lzham_uint32 lzham_decompress_deinit(lzham_decompress_state_ptr p)
{
return lzham::lzham_lib_decompress_deinit(p);
}
@ -38,9 +38,9 @@ extern "C" LZHAM_DLL_EXPORT lzham_decompress_status_t lzham_decompress(
return lzham::lzham_lib_decompress(p, pIn_buf, pIn_buf_size, pOut_buf, pOut_buf_size, no_more_input_bytes_flag);
}
extern "C" LZHAM_DLL_EXPORT lzham_decompress_status_t lzham_decompress_memory(const lzham_decompress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32 *pCrc32)
extern "C" LZHAM_DLL_EXPORT lzham_decompress_status_t lzham_decompress_memory(const lzham_decompress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32)
{
return lzham::lzham_lib_decompress_memory(pParams, pDst_buf, pDst_len, pSrc_buf, src_len, pAdler32, pCrc32);
return lzham::lzham_lib_decompress_memory(pParams, pDst_buf, pDst_len, pSrc_buf, src_len, pAdler32);
}
extern "C" LZHAM_DLL_EXPORT lzham_compress_state_ptr lzham_compress_init(const lzham_compress_params *pParams)
@ -53,7 +53,7 @@ extern "C" LZHAM_DLL_EXPORT lzham_compress_state_ptr lzham_compress_reinit(lzham
return lzham::lzham_lib_compress_reinit(p);
}
extern "C" LZHAM_DLL_EXPORT lzham_compress_checksums lzham_compress_deinit(lzham_compress_state_ptr p)
extern "C" LZHAM_DLL_EXPORT lzham_uint32 lzham_compress_deinit(lzham_compress_state_ptr p)
{
return lzham::lzham_lib_compress_deinit(p);
}
@ -76,9 +76,9 @@ extern "C" LZHAM_DLL_EXPORT lzham_compress_status_t lzham_compress2(
return lzham::lzham_lib_compress2(p, pIn_buf, pIn_buf_size, pOut_buf, pOut_buf_size, flush_type);
}
extern "C" LZHAM_DLL_EXPORT lzham_compress_status_t lzham_compress_memory(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32 * pCrc32)
extern "C" LZHAM_DLL_EXPORT lzham_compress_status_t lzham_compress_memory(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32)
{
return lzham::lzham_lib_compress_memory(pParams, pDst_buf, pDst_len, pSrc_buf, src_len, pAdler32, pCrc32);
return lzham::lzham_lib_compress_memory(pParams, pDst_buf, pDst_len, pSrc_buf, src_len, pAdler32);
}
// ----------------- zlib-style API's
@ -88,7 +88,7 @@ extern "C" LZHAM_DLL_EXPORT const char *lzham_z_version(void)
return LZHAM_Z_VERSION;
}
extern "C" lzham_z_ulong LZHAM_DLL_EXPORT lzham_z_adler32(lzham_z_ulong adler, const unsigned char *ptr, size_t buf_len)
extern "C" lzham_z_ulong LZHAM_DLL_EXPORT lzham_z_adler32(lzham_z_ulong adler, const lzham_uint8 *ptr, size_t buf_len)
{
return lzham::lzham_lib_z_adler32(adler, ptr, buf_len);
}

4
r5dev/thirdparty/lzham/lzhamcomp/lzham_comp.h vendored
View File

@ -9,7 +9,7 @@ namespace lzham
lzham_compress_state_ptr LZHAM_CDECL lzham_lib_compress_reinit(lzham_compress_state_ptr p);
lzham_compress_checksums LZHAM_CDECL lzham_lib_compress_deinit(lzham_compress_state_ptr p);
lzham_uint32 LZHAM_CDECL lzham_lib_compress_deinit(lzham_compress_state_ptr p);
lzham_compress_status_t LZHAM_CDECL lzham_lib_compress(
lzham_compress_state_ptr p,
@ -23,7 +23,7 @@ namespace lzham
lzham_uint8 *pOut_buf, size_t *pOut_buf_size,
lzham_flush_t flush_type);
lzham_compress_status_t LZHAM_CDECL lzham_lib_compress_memory(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32* pCrc32);
lzham_compress_status_t LZHAM_CDECL lzham_lib_compress_memory(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32);
int lzham_lib_z_deflateInit(lzham_z_streamp pStream, int level);
int lzham_lib_z_deflateInit2(lzham_z_streamp pStream, int level, int method, int window_bits, int mem_level, int strategy);

21
r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp.cpp vendored
View File

@ -150,18 +150,16 @@ namespace lzham
return pState;
}
lzham_compress_checksums LZHAM_CDECL lzham_lib_compress_deinit(lzham_compress_state_ptr p)
lzham_uint32 LZHAM_CDECL lzham_lib_compress_deinit(lzham_compress_state_ptr p)
{
lzham_compress_checksums checksums{};
lzham_compress_state *pState = static_cast<lzham_compress_state *>(p);
if (!pState)
return checksums;
return 0;
checksums.adler32 = pState->m_compressor.get_src_adler32();
checksums.crc32 = pState->m_compressor.get_src_crc32();
uint32 adler32 = pState->m_compressor.get_src_adler32();
lzham_delete(pState);
return checksums;
return adler32;
}
lzham_compress_status_t LZHAM_CDECL lzham_lib_compress(
@ -296,7 +294,7 @@ namespace lzham
return pState->m_status;
}
lzham_compress_status_t LZHAM_CDECL lzham_lib_compress_memory(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32 *pCrc32)
lzham_compress_status_t LZHAM_CDECL lzham_lib_compress_memory(const lzham_compress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32)
{
if ((!pParams) || (!pDst_len))
return LZHAM_COMP_STATUS_INVALID_PARAMETER;
@ -368,8 +366,6 @@ namespace lzham
if (pAdler32)
*pAdler32 = pCompressor->get_src_adler32();
if (pCrc32)
*pCrc32 = pCompressor->get_src_crc32();
if (comp_data.size() == dst_buf_size)
{
@ -523,7 +519,6 @@ namespace lzham
pStream->total_out += (uint)out_bytes;
pStream->adler32 = pState->m_compressor.get_src_adler32();
pStream->crc32 = pState->m_compressor.get_src_crc32();
if (status >= LZHAM_COMP_STATUS_FIRST_FAILURE_CODE)
{
@ -555,11 +550,7 @@ namespace lzham
lzham_compress_state_ptr pComp = (lzham_compress_state_ptr)pStream->state;
if (pComp)
{
lzham_compress_checksums checksums = lzham_lib_compress_deinit(pComp);
pStream->adler32 = checksums.adler32;
pStream->crc32 = checksums.crc32;
pStream->adler32 = lzham_lib_compress_deinit(pComp);
pStream->state = NULL;
}

7
r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp_internal.cpp vendored
View File

@ -69,7 +69,6 @@ namespace lzham
lzcompressor::lzcompressor() :
m_src_size(-1),
m_src_adler32(0),
m_src_crc32(0),
m_step(0),
m_block_start_dict_ofs(0),
m_block_index(0),
@ -298,7 +297,6 @@ namespace lzham
m_codec.clear();
m_src_size = -1;
m_src_adler32 = cInitAdler32;
m_src_crc32 = cInitCRC32;
m_block_buf.clear();
m_comp_buf.clear();
@ -341,7 +339,6 @@ namespace lzham
m_stats.clear();
m_src_size = 0;
m_src_adler32 = cInitAdler32;
m_src_crc32 = cInitCRC32;
m_block_buf.try_resize(0);
m_comp_buf.try_resize(0);
@ -572,9 +569,6 @@ namespace lzham
if (!m_codec.encode_bits(m_src_adler32, 32))
return false;
if (!m_codec.encode_bits(m_src_crc32, 32))
return false;
if (!m_codec.stop_encoding(true))
return false;
@ -1547,7 +1541,6 @@ namespace lzham
m_start_of_block_state = m_state;
m_src_adler32 = adler32(pBuf, buf_len, m_src_adler32);
m_src_crc32 = crc32(pBuf, buf_len, m_src_crc32);
m_block_start_dict_ofs = m_accel.get_lookahead_pos() & (m_accel.get_max_dict_size() - 1);

2
r5dev/thirdparty/lzham/lzhamcomp/lzham_lzcomp_internal.h vendored
View File

@ -91,7 +91,6 @@ namespace lzham
byte_vec& get_compressed_data() { return m_comp_buf; }
uint32 get_src_adler32() const { return m_src_adler32; }
uint32 get_src_crc32() const { return m_src_crc32; }
private:
class state;
@ -346,7 +345,6 @@ namespace lzham
int64 m_src_size;
uint32 m_src_adler32;
uint32 m_src_crc32;
search_accelerator m_accel;

6
r5dev/thirdparty/lzham/lzhamdecomp/lzham_decomp.h vendored
View File

@ -11,7 +11,7 @@ namespace lzham
lzham_decompress_state_ptr LZHAM_CDECL lzham_lib_decompress_reinit(lzham_decompress_state_ptr pState, const lzham_decompress_params *pParams);
lzham_decompress_checksums LZHAM_CDECL lzham_lib_decompress_deinit(lzham_decompress_state_ptr pState);
lzham_uint32 LZHAM_CDECL lzham_lib_decompress_deinit(lzham_decompress_state_ptr pState);
lzham_decompress_status_t LZHAM_CDECL lzham_lib_decompress(
lzham_decompress_state_ptr pState,
@ -21,7 +21,7 @@ namespace lzham
lzham_decompress_status_t LZHAM_CDECL lzham_lib_decompress_memory(const lzham_decompress_params *pParams,
lzham_uint8* pDst_buf, size_t *pDst_len,
const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32* pCrc32);
const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32);
int LZHAM_CDECL lzham_lib_z_inflateInit2(lzham_z_streamp pStream, int window_bits);
int LZHAM_CDECL lzham_lib_z_inflateInit(lzham_z_streamp pStream);
@ -31,7 +31,7 @@ namespace lzham
int LZHAM_CDECL lzham_lib_z_uncompress(unsigned char *pDest, lzham_z_ulong *pDest_len, const unsigned char *pSource, lzham_z_ulong source_len);
const char * LZHAM_CDECL lzham_lib_z_error(int err);
lzham_z_ulong LZHAM_CDECL lzham_lib_z_adler32(lzham_z_ulong adler, const lzham_uint8*ptr, size_t buf_len);
lzham_z_ulong LZHAM_CDECL lzham_lib_z_adler32(lzham_z_ulong adler, const lzham_uint8 *ptr, size_t buf_len);
lzham_z_ulong LZHAM_CDECL lzham_lib_z_crc32(lzham_z_ulong crc, const lzham_uint8 *ptr, size_t buf_len);
} // namespace lzham

46
r5dev/thirdparty/lzham/lzhamdecomp/lzham_lzdecomp.cpp vendored
View File

@ -43,7 +43,6 @@ namespace lzham
uint8 *m_pRaw_decomp_buf;
uint8 *m_pDecomp_buf;
uint32 m_decomp_adler32;
uint32 m_decomp_crc32;
const uint8 *m_pIn_buf;
size_t *m_pIn_buf_size;
@ -105,7 +104,6 @@ namespace lzham
uint m_seed_bytes_to_ignore_when_flushing;
uint m_file_src_file_adler32;
uint m_file_src_file_crc32;
uint m_rep_lit0;
uint m_match_len;
@ -193,7 +191,6 @@ namespace lzham
size_t bytes_to_copy = LZHAM_MIN((size_t)(m_flush_n - copy_ofs), cBytesToMemCpyPerIteration); \
LZHAM_MEMCPY(m_pOut_buf + copy_ofs, m_pFlush_src + copy_ofs, bytes_to_copy); \
m_decomp_adler32 = adler32(m_pFlush_src + copy_ofs, bytes_to_copy, m_decomp_adler32); \
m_decomp_crc32 = crc32(m_pFlush_src + copy_ofs, bytes_to_copy, m_decomp_crc32); \
copy_ofs += bytes_to_copy; \
} \
} \
@ -239,7 +236,6 @@ namespace lzham
m_pOrig_out_buf = NULL;
m_orig_out_buf_size = 0;
m_decomp_adler32 = cInitAdler32;
m_decomp_crc32 = cInitCRC32;
m_seed_bytes_to_ignore_when_flushing = 0;
m_z_last_status = LZHAM_DECOMP_STATUS_NOT_FINISHED;
@ -1142,23 +1138,6 @@ namespace lzham
{
m_decomp_adler32 = m_file_src_file_adler32;
}
if (m_params.m_decompress_flags & LZHAM_DECOMP_FLAG_COMPUTE_CRC32)
{
if (unbuffered)
{
m_decomp_crc32 = crc32(pDst, dst_ofs, cInitCRC32);
}
//if (m_file_src_file_crc32 != m_decomp_crc32)
//{
// printf("m_file_src_file_crc32 %zX\n", m_file_src_file_crc32);
// m_status = LZHAM_DECOMP_STATUS_FAILED_CRC32;
//}
}
else
{
m_decomp_crc32 = m_file_src_file_crc32;
}
}
LZHAM_SYMBOL_CODEC_DECODE_END(codec);
@ -1273,20 +1252,18 @@ namespace lzham
return pState;
}
lzham_decompress_checksums LZHAM_CDECL lzham_lib_decompress_deinit(lzham_decompress_state_ptr p)
lzham_uint32 LZHAM_CDECL lzham_lib_decompress_deinit(lzham_decompress_state_ptr p)
{
lzham_decompress_checksums checksums{};
lzham_decompressor *pState = static_cast<lzham_decompressor *>(p);
if (!pState)
return checksums;
return 0;
checksums.adler32 = pState->m_decomp_adler32;
checksums.crc32 = pState->m_decomp_crc32;
uint32 adler32 = pState->m_decomp_adler32;
lzham_free(pState->m_pRaw_decomp_buf);
lzham_delete(pState);
return checksums;
return adler32;
}
lzham_decompress_status_t LZHAM_CDECL lzham_lib_decompress(
@ -1343,7 +1320,7 @@ namespace lzham
return status;
}
lzham_decompress_status_t LZHAM_CDECL lzham_lib_decompress_memory(const lzham_decompress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32, lzham_uint32 *pCrc32)
lzham_decompress_status_t LZHAM_CDECL lzham_lib_decompress_memory(const lzham_decompress_params *pParams, lzham_uint8* pDst_buf, size_t *pDst_len, const lzham_uint8* pSrc_buf, size_t src_len, lzham_uint32 *pAdler32)
{
if (!pParams)
return LZHAM_DECOMP_STATUS_INVALID_PARAMETER;
@ -1356,13 +1333,10 @@ namespace lzham
return LZHAM_DECOMP_STATUS_FAILED_INITIALIZING;
lzham_decompress_status_t status = lzham_lib_decompress(pState, pSrc_buf, &src_len, pDst_buf, pDst_len, true);
lzham_decompress_checksums checksums = lzham_lib_decompress_deinit(pState);
lzham_uint32 adler32 = lzham_lib_decompress_deinit(pState);
if (pAdler32)
*pAdler32 = checksums.adler32;
if (pCrc32)
*pCrc32 = checksums.crc32;
*pAdler32 = adler32;
return status;
}
@ -1468,7 +1442,6 @@ namespace lzham
pStream->avail_in -= (uint)in_bytes;
pStream->total_in += (uint)in_bytes;
pStream->adler32 = pDecomp->m_decomp_adler32;
pStream->crc32 = pDecomp->m_decomp_crc32;
pStream->next_out += (uint)out_bytes;
pStream->avail_out -= (uint)out_bytes;
@ -1510,10 +1483,7 @@ namespace lzham
lzham_decompress_state_ptr pState = static_cast<lzham_decompress_state_ptr>(pStream->state);
if (pState)
{
lzham_decompress_checksums checksums = lzham_lib_decompress_deinit(pState);
pStream->adler32 = checksums.adler32;
pStream->crc32 = checksums.crc32;
pStream->adler32 = lzham_lib_decompress_deinit(pState);
pStream->state = NULL;
}

51
r5dev/vpklib/packedstore.cpp
View File

@ -47,7 +47,7 @@ void CPackedStore::InitLzDecompParams(void)
{
/*| PARAMETERS ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||*/
m_lzDecompParams.m_dict_size_log2 = VPK_DICT_SIZE;
m_lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED | lzham_decompress_flags::LZHAM_DECOMP_FLAG_COMPUTE_CRC32;
m_lzDecompParams.m_decompress_flags = lzham_decompress_flags::LZHAM_DECOMP_FLAG_OUTPUT_UNBUFFERED;
m_lzDecompParams.m_struct_size = sizeof(lzham_decompress_params);
}
@ -317,40 +317,12 @@ void CPackedStore::BuildManifest(const vector<VPKEntryBlock_t>& vBlock, const st
FileSystem()->WriteFile(svPathOut.c_str(), "GAME", uBuf);
}
//-----------------------------------------------------------------------------
// Purpose: validates extraction result with precomputed ADLER32 hash
// Input : &svAssetPath -
//-----------------------------------------------------------------------------
void CPackedStore::ValidateAdler32PostDecomp(const string& svAssetPath)
{
FileHandle_t hAsset = FileSystem()->Open(svAssetPath.c_str(), "rb", "GAME");
if (!hAsset)
{
Error(eDLL_T::FS, NO_ERROR, "%s - Unable to open '%s' (insufficient rights?)\n", __FUNCTION__, svAssetPath.c_str());
return;
}
uint32_t nLen = FileSystem()->Size(hAsset);
uint8_t* pBuf = MemAllocSingleton()->Alloc<uint8_t>(nLen);
FileSystem()->Read(pBuf, nLen, hAsset);
FileSystem()->Close(hAsset);
m_nAdler32 = adler32::update(NULL, pBuf, nLen);
MemAllocSingleton()->Free(pBuf);
if (m_nAdler32 != m_nAdler32_Internal)
{
Warning(eDLL_T::FS, "Computed checksum '0x%lX' doesn't match expected checksum '0x%lX'. File may be corrupt!\n", m_nAdler32, m_nAdler32_Internal);
m_nAdler32 = NULL;
m_nAdler32_Internal = NULL;
}
}
//-----------------------------------------------------------------------------
// Purpose: validates extraction result with precomputed CRC32 hash
// Input : &svAssetPath -
// : nFileCRC -
//-----------------------------------------------------------------------------
void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetPath)
void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetPath, const uint32_t nFileCRC)
{
FileHandle_t hAsset = FileSystem()->Open(svAssetPath.c_str(), "rb", "GAME");
if (!hAsset)
@ -364,14 +336,13 @@ void CPackedStore::ValidateCRC32PostDecomp(const string& svAssetPath)
FileSystem()->Read(pBuf, nLen, hAsset);
FileSystem()->Close(hAsset);
m_nCrc32 = crc32::update(NULL, pBuf, nLen);
uint32_t nCrc32 = crc32::update(NULL, pBuf, nLen);
MemAllocSingleton()->Free(pBuf);
if (m_nCrc32 != m_nCrc32_Internal)
if (nCrc32 != nFileCRC)
{
Warning(eDLL_T::FS, "Computed checksum '0x%lX' doesn't match expected checksum '0x%lX'. File may be corrupt!\n", m_nCrc32, m_nCrc32_Internal);
m_nCrc32 = NULL;
m_nCrc32_Internal = NULL;
Warning(eDLL_T::FS, "Computed checksum '0x%lX' doesn't match expected checksum '0x%lX'. File may be corrupt!\n", nCrc32, nFileCRC);
nCrc32 = NULL;
}
}
@ -463,7 +434,7 @@ void CPackedStore::PackWorkspace(const VPKPair_t& vPair, const string& svWorkspa
if (vEntryValue.m_bUseCompression)
{
m_lzCompStatus = lzham_compress_memory(&m_lzCompParams, s_EntryBuf, &vDescriptor.m_nCompressedSize, s_EntryBuf,
vDescriptor.m_nUncompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
vDescriptor.m_nUncompressedSize, nullptr);
if (m_lzCompStatus != lzham_compress_status_t::LZHAM_COMP_STATUS_SUCCESS)
{
@ -562,7 +533,7 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDirectory, const string& svW
break; // Corrupt or invalid chunk descriptor.
m_lzDecompStatus = lzham_decompress_memory(&m_lzDecompParams, s_DecompBuf,
&nDstLen, s_EntryBuf, vChunk.m_nCompressedSize, &m_nAdler32_Internal, &m_nCrc32_Internal);
&nDstLen, s_EntryBuf, vChunk.m_nCompressedSize, nullptr);
if (m_lzDecompStatus != lzham_decompress_status_t::LZHAM_DECOMP_STATUS_SUCCESS)
{
@ -584,9 +555,7 @@ void CPackedStore::UnpackWorkspace(const VPKDir_t& vDirectory, const string& svW
if (m_nChunkCount == vEntryBlock.m_vFragments.size()) // Only validate after last entry in block had been written.
{
m_nChunkCount = NULL;
m_nCrc32_Internal = vEntryBlock.m_nFileCRC;
ValidateCRC32PostDecomp(svFilePath);
ValidateCRC32PostDecomp(svFilePath, vEntryBlock.m_nFileCRC);
}
}
}

8
r5dev/vpklib/packedstore.h
View File

@ -156,16 +156,10 @@ public:
void PackWorkspace(const VPKPair_t& vPair, const string& svWorkspace, const string& svBuildPath, bool bManifestOnly);
void UnpackWorkspace(const VPKDir_t& vDirectory, const string& svWorkspace = "");
void ValidateAdler32PostDecomp(const string& svAssetPath);
void ValidateCRC32PostDecomp(const string& svAssetPath);
void ValidateCRC32PostDecomp(const string& svAssetPath, const uint32_t nFileCRC);
private:
size_t m_nChunkCount; // The number of fragments for this asset.
lzham_uint32 m_nAdler32_Internal; // Internal operation Adler32 file checksum.
lzham_uint32 m_nAdler32; // Pre/post operation Adler32 file checksum.
lzham_uint32 m_nCrc32_Internal; // Internal operation Crc32 file checksum.
lzham_uint32 m_nCrc32; // Pre/post operation Crc32 file checksum.
lzham_compress_params m_lzCompParams; // LZham decompression parameters.
lzham_compress_status_t m_lzCompStatus; // LZham compression status.
lzham_decompress_params m_lzDecompParams; // LZham decompression parameters.