xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-07 20:06:59 -06:00
Code Issues Packages Projects Releases Wiki Activity

update of zstd to v1.1.1

Browse Source
This commit is contained in:
Tino Reichardt
2016-11-19 11:32:02 +01:00
parent 861406f4cd
commit fe3a65bfbd
15 changed files with 502 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
C/zstd/entropy_common.c
View File

@@ -168,9 +168,11 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
{ {
U32 weightTotal; U32 weightTotal;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */ if (iSize >= 128) { /* special header */
@@ -198,6 +200,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} } } }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1; { U32 const tableLog = BIT_highbit32(weightTotal) + 1;

59
C/zstd/error_public.h
View File

@@ -1,59 +0,0 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*===== dependency =====*/
#include <stddef.h> /* size_t */
/*-****************************************
* error codes list
******************************************/
typedef enum {
ZSTD_error_no_error,
ZSTD_error_GENERIC,
ZSTD_error_prefix_unknown,
ZSTD_error_version_unsupported,
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
ZSTD_error_stage_wrong,
ZSTD_error_dstSize_tooSmall,
ZSTD_error_srcSize_wrong,
ZSTD_error_corruption_detected,
ZSTD_error_checksum_wrong,
ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,
ZSTD_error_dictionary_corrupted,
ZSTD_error_dictionary_wrong,
ZSTD_error_maxCode
} ZSTD_ErrorCode;
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */

9
C/zstd/huf_compress.c
View File

@@ -155,13 +155,14 @@ size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, si
} } } }
/* fill val */ /* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; { U16 nbPerRank[HUF_TABLELOG_MAX+2] = {0}; /* support w=0=>n=tableLog+1 */
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; U16 valPerRank[HUF_TABLELOG_MAX+2] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; } { U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */ /* determine stating value per rank */
valPerRank[tableLog+1] = 0; /* for w==0 */
{ U16 min = 0; { U16 min = 0;
U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) { U32 n; for (n=tableLog; n>0; n--) { /* start at n=tablelog <-> w=1 */
valPerRank[n] = min; /* get starting value within each rank */ valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n]; min += nbPerRank[n];
min >>= 1; min >>= 1;
} } } }

202
C/zstd/zstd.h
View File

@@ -1,4 +1,4 @@
/** /*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved. * All rights reserved.
* *
@@ -14,12 +14,12 @@
extern "C" { extern "C" {
#endif #endif
/*====== Dependency ======*/ /* ====== Dependency ======*/
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/*====== Export for Windows ======*/ /* ====== Export for Windows ======*/
/*! /*
* ZSTD_DLL_EXPORT : * ZSTD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL * Enable exporting of functions when building a Windows DLL
*/ */
@@ -30,7 +30,29 @@ extern "C" {
#endif #endif
/*======= Version =======*/ /*******************************************************************************************************
Introduction
zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
Compression can be done in:
- a single step (described as Simple API)
- a single step, reusing a context (described as Explicit memory management)
- unbounded multiple steps (described as Streaming compression)
The compression ratio achievable on small data can be highly improved using compression with a dictionary in:
- a single step (described as Simple dictionary API)
- a single step, reusing a dictionary (described as Fast dictionary API)
Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h.
These APIs shall never be used with a dynamic library.
They are not "stable", their definition may change in the future. Only static linking is allowed.
*********************************************************************************************************/
/*------ Version ------*/
ZSTDLIB_API unsigned ZSTD_versionNumber (void); /**< returns version number of ZSTD */
#define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1 #define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 1 #define ZSTD_VERSION_RELEASE 1
@@ -41,10 +63,9 @@ extern "C" {
#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION)
#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE)
ZSTDLIB_API unsigned ZSTD_versionNumber (void);
/* ************************************* /***************************************
* Simple API * Simple API
***************************************/ ***************************************/
/*! ZSTD_compress() : /*! ZSTD_compress() :
@@ -91,29 +112,33 @@ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `siz
ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */
/*-************************************* /***************************************
* Explicit memory management * Explicit memory management
***************************************/ ***************************************/
/** Compression context */ /*= Compression context
* When compressing many messages / blocks,
* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
* This will make the situation much easier for the system's memory.
* Use one context per thread for parallel execution in multi-threaded environments. */
typedef struct ZSTD_CCtx_s ZSTD_CCtx; typedef struct ZSTD_CCtx_s ZSTD_CCtx;
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void);
ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
/** ZSTD_compressCCtx() : /*! ZSTD_compressCCtx() :
Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */ Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()) */
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
/** Decompression context */ /*= Decompression context */
typedef struct ZSTD_DCtx_s ZSTD_DCtx; typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
/** ZSTD_decompressDCtx() : /*! ZSTD_decompressDCtx() :
* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */
ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-************************ /**************************
* Simple dictionary API * Simple dictionary API
***************************/ ***************************/
/*! ZSTD_compress_usingDict() : /*! ZSTD_compress_usingDict() :
@@ -135,14 +160,20 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
const void* dict,size_t dictSize); const void* dict,size_t dictSize);
/*-************************** /****************************
* Fast Dictionary API * Fast dictionary API
****************************/ ****************************/
/*! ZSTD_createCDict() :
* Create a digested dictionary, ready to start compression operation without startup delay.
* `dict` can be released after ZSTD_CDict creation */
typedef struct ZSTD_CDict_s ZSTD_CDict; typedef struct ZSTD_CDict_s ZSTD_CDict;
/*! ZSTD_createCDict() :
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
* `dict` can be released after ZSTD_CDict creation */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel);
/*! ZSTD_freeCDict() :
* Function frees memory allocated by ZSTD_createCDict() */
ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict);
/*! ZSTD_compress_usingCDict() : /*! ZSTD_compress_usingCDict() :
@@ -154,11 +185,16 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const ZSTD_CDict* cdict); const ZSTD_CDict* cdict);
typedef struct ZSTD_DDict_s ZSTD_DDict;
/*! ZSTD_createDDict() : /*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay. * Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation */ * `dict` can be released after creation */
typedef struct ZSTD_DDict_s ZSTD_DDict;
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize);
/*! ZSTD_freeDDict() :
* Function frees memory allocated with ZSTD_createDDict() */
ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict);
/*! ZSTD_decompress_usingDDict() : /*! ZSTD_decompress_usingDDict() :
@@ -170,7 +206,7 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict); const ZSTD_DDict* ddict);
/*-************************** /****************************
* Streaming * Streaming
****************************/ ****************************/
@@ -187,16 +223,18 @@ typedef struct ZSTD_outBuffer_s {
} ZSTD_outBuffer; } ZSTD_outBuffer;
/*====== streaming compression ======*/
/*-*********************************************************************** /*-***********************************************************************
* Streaming compression - howto * Streaming compression - HowTo
* *
* A ZSTD_CStream object is required to track streaming operation. * A ZSTD_CStream object is required to track streaming operation.
* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources.
* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. * ZSTD_CStream objects can be reused multiple times on consecutive compression operations.
* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively,
* since it will play nicer with system's memory, by re-using already allocated memory.
* Use one separate ZSTD_CStream per thread for parallel execution.
* *
* Start by initializing ZSTD_CStream. * Start a new compression by initializing ZSTD_CStream.
* Use ZSTD_initCStream() to start a new compression operation. * Use ZSTD_initCStream() to start a new compression operation.
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. * Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
* *
@@ -225,23 +263,22 @@ typedef struct ZSTD_outBuffer_s {
* *
* *******************************************************************/ * *******************************************************************/
/*===== Streaming compression functions ======*/
typedef struct ZSTD_CStream_s ZSTD_CStream; typedef struct ZSTD_CStream_s ZSTD_CStream;
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
/*====== decompression ======*/
/*-*************************************************************************** /*-***************************************************************************
* Streaming decompression howto * Streaming decompression - HowTo
* *
* A ZSTD_DStream object is required to track streaming operations. * A ZSTD_DStream object is required to track streaming operations.
* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources.
@@ -262,28 +299,29 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
* The return value is a suggested next input size (just an hint, to help latency). * The return value is a suggested next input size (just an hint, to help latency).
* *******************************************************************************/ * *******************************************************************************/
/*===== Streaming decompression functions =====*/
typedef struct ZSTD_DStream_s ZSTD_DStream; typedef struct ZSTD_DStream_s ZSTD_DStream;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
#ifdef ZSTD_STATIC_LINKING_ONLY #ifdef ZSTD_STATIC_LINKING_ONLY
/* ==================================================================================== /****************************************************************************************
* START OF ADVANCED AND EXPERIMENTAL FUNCTIONS
* The definitions in this section are considered experimental. * The definitions in this section are considered experimental.
* They should never be used with a dynamic library, as they may change in the future. * They should never be used with a dynamic library, as they may change in the future.
* They are provided for advanced usages. * They are provided for advanced usages.
* Use them only in association with static linking. * Use them only in association with static linking.
* ==================================================================================== */ * ***************************************************************************************/
/*--- Constants ---*/ /* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
@@ -310,8 +348,8 @@ static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
/*--- Types ---*/ /*--- Advanced types ---*/
typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /* from faster to stronger */ typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */
typedef struct { typedef struct {
unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */
@@ -334,13 +372,13 @@ typedef struct {
ZSTD_frameParameters fParams; ZSTD_frameParameters fParams;
} ZSTD_parameters; } ZSTD_parameters;
/* custom memory allocation functions */ /*= Custom memory allocation functions */
typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address); typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
/*-************************************* /***************************************
* Advanced compression functions * Advanced compression functions
***************************************/ ***************************************/
/*! ZSTD_estimateCCtxSize() : /*! ZSTD_estimateCCtxSize() :
@@ -393,7 +431,7 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
ZSTD_parameters params); ZSTD_parameters params);
/*--- Advanced Decompression functions ---*/ /*--- Advanced decompression functions ---*/
/*! ZSTD_estimateDCtxSize() : /*! ZSTD_estimateDCtxSize() :
* Gives the potential amount of memory allocated to create a ZSTD_DCtx */ * Gives the potential amount of memory allocated to create a ZSTD_DCtx */
@@ -412,47 +450,41 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/* ****************************************************************** /********************************************************************
* Advanced Streaming functions * Advanced streaming functions
********************************************************************/ ********************************************************************/
/*====== compression ======*/ /*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
/*====== decompression ======*/ /*===== Advanced Streaming decompression functions =====*/
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
/* ****************************************************************** /*********************************************************************
* Buffer-less and synchronous inner streaming functions * Buffer-less and synchronous inner streaming functions
********************************************************************/ *
/* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. * This is an advanced API, giving full control over buffer management, for users which need direct control over memory.
* But it's also a complex one, with many restrictions (documented below). * But it's also a complex one, with many restrictions (documented below).
* Prefer using normal streaming API for an easier experience */ * Prefer using normal streaming API for an easier experience
********************************************************************* */
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); /**
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); Buffer-less streaming compression (synchronous mode)
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*
A ZSTD_CCtx object is required to track streaming operations. A ZSTD_CCtx object is required to track streaming operations.
Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource.
ZSTD_CCtx object can be re-used multiple times within successive compression operations. ZSTD_CCtx object can be re-used multiple times within successive compression operations.
@@ -481,26 +513,17 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame.
*/ */
typedef struct { /*===== Buffer-less streaming compression functions =====*/
unsigned long long frameContentSize; ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
unsigned windowSize; ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
unsigned dictID; ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
unsigned checksumFlag; ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
} ZSTD_frameParams; ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); /*-
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/*
Buffer-less streaming decompression (synchronous mode) Buffer-less streaming decompression (synchronous mode)
A ZSTD_DCtx object is required to track streaming operations. A ZSTD_DCtx object is required to track streaming operations.
@@ -557,11 +580,27 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
It also returns Frame Size as fparamsPtr->frameContentSize. It also returns Frame Size as fparamsPtr->frameContentSize.
*/ */
typedef struct {
unsigned long long frameContentSize;
unsigned windowSize;
unsigned dictID;
unsigned checksumFlag;
} ZSTD_frameParams;
/* ************************************** /*===== Buffer-less streaming decompression functions =====*/
* Block functions ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */
****************************************/ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx);
/*! Block functions produce and decode raw zstd blocks, without frame metadata. ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx);
ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx);
ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e;
ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
/**
Block functions
Block functions produce and decode raw zstd blocks, without frame metadata.
Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes).
User will have to take in charge required information to regenerate data, such as compressed and content sizes. User will have to take in charge required information to regenerate data, such as compressed and content sizes.
@@ -585,6 +624,7 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx);
*/ */
#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ #define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */
/*===== Raw zstd block functions =====*/
ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx);
ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);

165
C/zstd/zstd_compress.c
View File

@@ -122,6 +122,11 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface *
return &(ctx->seqStore); return &(ctx->seqStore);
} }
static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx)
{
return cctx->params;
}
/** ZSTD_checkParams() : /** ZSTD_checkParams() :
ensure param values remain within authorized range. ensure param values remain within authorized range.
@@ -137,7 +142,7 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1;
CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); }
CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
if ((U32)(cParams.strategy) > (U32)ZSTD_btopt) return ERROR(compressionParameter_unsupported); if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported);
return 0; return 0;
} }
@@ -160,7 +165,7 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} } } }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt); { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt) | (cPar.strategy == ZSTD_btopt2);
U32 const maxChainLog = cPar.windowLog+btPlus; U32 const maxChainLog = cPar.windowLog+btPlus;
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
@@ -186,7 +191,7 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((cParams.strategy == ZSTD_btopt) ? optSpace : 0); + (((cParams.strategy == ZSTD_btopt) || (cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
return sizeof(ZSTD_CCtx) + neededSpace; return sizeof(ZSTD_CCtx) + neededSpace;
} }
@@ -246,7 +251,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); + (((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) ? optSpace : 0);
if (zc->workSpaceSize < neededSpace) { if (zc->workSpaceSize < neededSpace) {
ZSTD_free(zc->workSpace, zc->customMem); ZSTD_free(zc->workSpace, zc->customMem);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
@@ -276,7 +281,7 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
zc->frameContentSize = frameContentSize; zc->frameContentSize = frameContentSize;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; } { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; }
if (params.cParams.strategy == ZSTD_btopt) { if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) {
zc->seqStore.litFreq = (U32*)ptr; zc->seqStore.litFreq = (U32*)ptr;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
@@ -377,7 +382,7 @@ static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue)
* Block entropic compression * Block entropic compression
*********************************************************/ *********************************************************/
/* See zstd_compression_format.md for detailed format description */ /* See doc/zstd_compression_format.md for detailed format description */
size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
@@ -2170,7 +2175,17 @@ static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src,
static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize); ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0);
#else
(void)ctx; (void)src; (void)srcSize;
return;
#endif
}
static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
#ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx; (void)src; (void)srcSize;
return; return;
@@ -2180,7 +2195,17 @@ static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t src
static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{ {
#ifdef ZSTD_OPT_H_91842398743 #ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize); ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0);
#else
(void)ctx; (void)src; (void)srcSize;
return;
#endif
}
static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize)
{
#ifdef ZSTD_OPT_H_91842398743
ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1);
#else #else
(void)ctx; (void)src; (void)srcSize; (void)ctx; (void)src; (void)srcSize;
return; return;
@@ -2192,9 +2217,9 @@ typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t sr
static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict)
{ {
static const ZSTD_blockCompressor blockCompressor[2][7] = { static const ZSTD_blockCompressor blockCompressor[2][8] = {
{ ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt }, { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 },
{ ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict } { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict }
}; };
return blockCompressor[extDict][(U32)strat]; return blockCompressor[extDict][(U32)strat];
@@ -2247,7 +2272,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
/* preemptive overflow correction */ /* preemptive overflow correction */
if (cctx->lowLimit > (1<<30)) { if (cctx->lowLimit > (1<<30)) {
U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt); U32 const btplus = (cctx->params.cParams.strategy == ZSTD_btlazy2) | (cctx->params.cParams.strategy == ZSTD_btopt) | (cctx->params.cParams.strategy == ZSTD_btopt2);
U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1; U32 const chainMask = (1 << (cctx->params.cParams.chainLog - btplus)) - 1;
U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */); U32 const supLog = MAX(cctx->params.cParams.chainLog, 17 /* blockSize */);
U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */ U32 const newLowLimit = (cctx->lowLimit & chainMask) + (1 << supLog); /* preserve position % chainSize, ensure current-repcode doesn't underflow */
@@ -2436,6 +2461,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
case ZSTD_btlazy2: case ZSTD_btlazy2:
case ZSTD_btopt: case ZSTD_btopt:
case ZSTD_btopt2:
ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength);
break; break;
@@ -2448,14 +2474,28 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
} }
/* Dictionaries that assign zero probability to symbols that show up causes problems
when FSE encoding. Refuse dictionaries that assign zero probability to symbols
that we may encounter during compression.
NOTE: This behavior is not standard and could be improved in the future. */
static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) {
U32 s;
if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted);
for (s = 0; s <= maxSymbolValue; ++s) {
if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted);
}
return 0;
}
/* Dictionary format : /* Dictionary format :
Magic == ZSTD_DICT_MAGIC (4 bytes) Magic == ZSTD_DICT_MAGIC (4 bytes)
HUF_writeCTable(256) HUF_writeCTable(256)
FSE_writeNCount(off) FSE_writeNCount(off)
FSE_writeNCount(ml) FSE_writeNCount(ml)
FSE_writeNCount(ll) FSE_writeNCount(ll)
RepOffsets RepOffsets
Dictionary content Dictionary content
*/ */
/*! ZSTD_loadDictEntropyStats() : /*! ZSTD_loadDictEntropyStats() :
@return : size read from dictionary @return : size read from dictionary
@@ -2464,32 +2504,41 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
{ {
const BYTE* dictPtr = (const BYTE*)dict; const BYTE* dictPtr = (const BYTE*)dict;
const BYTE* const dictEnd = dictPtr + dictSize; const BYTE* const dictEnd = dictPtr + dictSize;
short offcodeNCount[MaxOff+1];
unsigned offcodeMaxValue = MaxOff;
{ size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize); { size_t const hufHeaderSize = HUF_readCTable(cctx->hufTable, 255, dict, dictSize);
if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted);
dictPtr += hufHeaderSize; dictPtr += hufHeaderSize;
} }
{ short offcodeNCount[MaxOff+1]; { unsigned offcodeLog;
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
/* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */
CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; { short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
/* Every match length code must have non-zero probability */
CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML));
CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; { short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
/* Every literal length code must have non-zero probability */
CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL));
CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
@@ -2500,6 +2549,16 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12; dictPtr += 12;
{ U32 offcodeMax = MaxOff;
if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) {
U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */
/* Calculate minimum offset code required to represent maxOffset */
offcodeMax = ZSTD_highbit32(maxOffset);
}
/* Every possible supported offset <= dictContentSize + 128 KB must be representable */
CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff)));
}
cctx->flagStaticTables = 1; cctx->flagStaticTables = 1;
return dictPtr - (const BYTE*)dict; return dictPtr - (const BYTE*)dict;
} }
@@ -2725,6 +2784,10 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
} }
} }
static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
return ZSTD_getParamsFromCCtx(cdict->refContext);
}
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{ {
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
@@ -2761,7 +2824,8 @@ typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s { struct ZSTD_CStream_s {
ZSTD_CCtx* cctx; ZSTD_CCtx* cctx;
ZSTD_CDict* cdict; ZSTD_CDict* cdictLocal;
const ZSTD_CDict* cdict;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inToCompress; size_t inToCompress;
@@ -2775,6 +2839,7 @@ struct ZSTD_CStream_s {
ZSTD_cStreamStage stage; ZSTD_cStreamStage stage;
U32 checksum; U32 checksum;
U32 frameEnded; U32 frameEnded;
ZSTD_parameters params;
ZSTD_customMem customMem; ZSTD_customMem customMem;
}; /* typedef'd to ZSTD_CStream within "zstd.h" */ }; /* typedef'd to ZSTD_CStream within "zstd.h" */
@@ -2804,7 +2869,7 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
if (zcs==NULL) return 0; /* support free on NULL */ if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem; { ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->cctx); ZSTD_freeCCtx(zcs->cctx);
ZSTD_freeCDict(zcs->cdict); ZSTD_freeCDict(zcs->cdictLocal);
ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem); ZSTD_free(zcs, cMem);
@@ -2820,7 +2885,10 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{ {
CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)); if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
zcs->inToCompress = 0; zcs->inToCompress = 0;
zcs->inBuffPos = 0; zcs->inBuffPos = 0;
@@ -2852,15 +2920,28 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (zcs->outBuff == NULL) return ERROR(memory_allocation); if (zcs->outBuff == NULL) return ERROR(memory_allocation);
} }
ZSTD_freeCDict(zcs->cdict); if (dict) {
zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); ZSTD_freeCDict(zcs->cdictLocal);
if (zcs->cdict == NULL) return ERROR(memory_allocation); zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL;
zcs->checksum = params.fParams.checksumFlag > 0; zcs->checksum = params.fParams.checksumFlag > 0;
zcs->params = params;
return ZSTD_resetCStream(zcs, pledgedSrcSize); return ZSTD_resetCStream(zcs, pledgedSrcSize);
} }
/* note : cdict must outlive compression session */
size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
{
ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
zcs->cdict = cdict;
return initError;
}
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
{ {
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
@@ -2875,7 +2956,7 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{ {
if (zcs==NULL) return 0; /* support sizeof on NULL */ if (zcs==NULL) return 0; /* support sizeof on NULL */
return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize; return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize;
} }
/*====== Compression ======*/ /*====== Compression ======*/
@@ -3066,9 +3147,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */
{ 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */ { 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */
{ 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */
{ 25, 25, 23, 7, 3, 64, ZSTD_btopt }, /* level 20 */ { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */
{ 26, 26, 23, 7, 3,256, ZSTD_btopt }, /* level 21 */ { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */
{ 27, 27, 25, 9, 3,512, ZSTD_btopt }, /* level 22 */ { 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */
}, },
{ /* for srcSize <= 256 KB */ { /* for srcSize <= 256 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
@@ -3092,9 +3173,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/
{ 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/
{ 18, 19, 18, 11, 3,512, ZSTD_btopt }, /* level 20.*/ { 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/
{ 18, 19, 18, 12, 3,512, ZSTD_btopt }, /* level 21.*/ { 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/
{ 18, 19, 18, 13, 3,512, ZSTD_btopt }, /* level 22.*/ { 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/
}, },
{ /* for srcSize <= 128 KB */ { /* for srcSize <= 128 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
@@ -3118,9 +3199,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/
{ 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/
{ 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/
{ 17, 18, 17, 9, 3,256, ZSTD_btopt }, /* level 20.*/ { 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/
{ 17, 18, 17, 10, 3,256, ZSTD_btopt }, /* level 21.*/ { 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/
{ 17, 18, 17, 11, 3,512, ZSTD_btopt }, /* level 22.*/ { 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/
}, },
{ /* for srcSize <= 16 KB */ { /* for srcSize <= 16 KB */
/* W, C, H, S, L, T, strat */ /* W, C, H, S, L, T, strat */
@@ -3144,9 +3225,9 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV
{ 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/
{ 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/
{ 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/
{ 14, 15, 15, 8, 3,256, ZSTD_btopt }, /* level 20.*/ { 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/
{ 14, 15, 15, 9, 3,256, ZSTD_btopt }, /* level 21.*/ { 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/
{ 14, 15, 15, 10, 3,256, ZSTD_btopt }, /* level 22.*/ { 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/
}, },
}; };

152
C/zstd/zstd_decompress.c
View File

@@ -34,7 +34,7 @@
* Frames requiring more memory will be rejected. * Frames requiring more memory will be rejected.
*/ */
#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT
# define ZSTD_MAXWINDOWSIZE_DEFAULT (257 << 20) /* 257 MB */ # define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */
#endif #endif
@@ -111,7 +111,7 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ }; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */ size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); }
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
@@ -170,20 +170,22 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{ {
ZSTD_decompressBegin(dstDCtx); /* init */ ZSTD_decompressBegin(dstDCtx); /* init */
dstDCtx->dictEnd = srcDCtx->dictEnd; if (srcDCtx) { /* support refDCtx on NULL */
dstDCtx->vBase = srcDCtx->vBase; dstDCtx->dictEnd = srcDCtx->dictEnd;
dstDCtx->base = srcDCtx->base; dstDCtx->vBase = srcDCtx->vBase;
dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; dstDCtx->base = srcDCtx->base;
dstDCtx->dictID = srcDCtx->dictID; dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
dstDCtx->litEntropy = srcDCtx->litEntropy; dstDCtx->dictID = srcDCtx->dictID;
dstDCtx->fseEntropy = srcDCtx->fseEntropy; dstDCtx->litEntropy = srcDCtx->litEntropy;
dstDCtx->LLTptr = srcDCtx->LLTable; dstDCtx->fseEntropy = srcDCtx->fseEntropy;
dstDCtx->MLTptr = srcDCtx->MLTable; dstDCtx->LLTptr = srcDCtx->LLTable;
dstDCtx->OFTptr = srcDCtx->OFTable; dstDCtx->MLTptr = srcDCtx->MLTable;
dstDCtx->HUFptr = srcDCtx->hufTable; dstDCtx->OFTptr = srcDCtx->OFTable;
dstDCtx->rep[0] = srcDCtx->rep[0]; dstDCtx->HUFptr = srcDCtx->hufTable;
dstDCtx->rep[1] = srcDCtx->rep[1]; dstDCtx->rep[0] = srcDCtx->rep[0];
dstDCtx->rep[2] = srcDCtx->rep[2]; dstDCtx->rep[1] = srcDCtx->rep[1];
dstDCtx->rep[2] = srcDCtx->rep[2];
}
} }
@@ -191,7 +193,7 @@ static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
* Decompression section * Decompression section
***************************************************************/ ***************************************************************/
/* See compression format details in : zstd_compression_format.md */ /* See compression format details in : doc/zstd_compression_format.md */
/** ZSTD_frameHeaderSize() : /** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_prefix. * srcSize must be >= ZSTD_frameHeaderSize_prefix.
@@ -301,14 +303,16 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize)
/** ZSTD_decodeFrameHeader() : /** ZSTD_decodeFrameHeader() :
* `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * `headerSize` must be the size provided by ZSTD_frameHeaderSize().
* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */
static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t srcSize) static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize)
{ {
size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, srcSize); size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize);
if (ZSTD_isError(result)) return result; /* invalid header */
if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */
if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong);
if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0);
return result; return 0;
} }
@@ -710,10 +714,13 @@ size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
{ int nbSeq = *ip++; { int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; } if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) { if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) if (nbSeq == 0xFF) {
if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
else } else {
if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++; nbSeq = ((nbSeq-0x80)<<8) + *ip++;
}
} }
*nbSeqPtr = nbSeq; *nbSeqPtr = nbSeq;
} }
@@ -807,7 +814,8 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
if (ofCode <= 1) { if (ofCode <= 1) {
offset += (llCode==0); offset += (llCode==0);
if (offset) { if (offset) {
size_t const temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset];
temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */
if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1];
seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[1] = seqState->prevOffset[0];
seqState->prevOffset[0] = offset = temp; seqState->prevOffset[0] = offset = temp;
@@ -839,6 +847,53 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState)
} }
FORCE_NOINLINE
size_t ZSTD_execSequenceLast7(BYTE* op,
BYTE* const oend, seq_t sequence,
const BYTE** litPtr, const BYTE* const litLimit_w,
const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd)
{
BYTE* const oLitEnd = op + sequence.litLength;
size_t const sequenceLength = sequence.litLength + sequence.matchLength;
BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */
BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH;
const BYTE* const iLitEnd = *litPtr + sequence.litLength;
const BYTE* match = oLitEnd - sequence.offset;
/* check */
if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */
/* copy literals */
if (op < oend_w) {
ZSTD_wildcopy(op, *litPtr, oend_w - op);
*litPtr += oend_w - op;
op = oend_w;
}
while (op < oLitEnd) *op++ = *(*litPtr)++;
/* copy Match */
if (sequence.offset > (size_t)(oLitEnd - base)) {
/* offset beyond prefix */
if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected);
match = dictEnd - (base-match);
if (match + sequence.matchLength <= dictEnd) {
memmove(oLitEnd, match, sequence.matchLength);
return sequenceLength;
}
/* span extDict & currentPrefixSegment */
{ size_t const length1 = dictEnd - match;
memmove(oLitEnd, match, length1);
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
} }
while (op < oMatchEnd) *op++ = *match++;
return sequenceLength;
}
FORCE_INLINE FORCE_INLINE
size_t ZSTD_execSequence(BYTE* op, size_t ZSTD_execSequence(BYTE* op,
BYTE* const oend, seq_t sequence, BYTE* const oend, seq_t sequence,
@@ -853,8 +908,9 @@ size_t ZSTD_execSequence(BYTE* op,
const BYTE* match = oLitEnd - sequence.offset; const BYTE* match = oLitEnd - sequence.offset;
/* check */ /* check */
if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */
if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */ if (iLitEnd > litLimit_w) return ERROR(corruption_detected); /* over-read beyond lit buffer */
if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit_w, base, vBase, dictEnd);
/* copy Literals */ /* copy Literals */
ZSTD_copy8(op, *litPtr); ZSTD_copy8(op, *litPtr);
@@ -879,7 +935,8 @@ size_t ZSTD_execSequence(BYTE* op,
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_w) { if (op > oend_w) {
memmove(op, match, sequence.matchLength); U32 i;
for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i];
return sequenceLength; return sequenceLength;
} }
} } } }
@@ -1315,25 +1372,28 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
} }
{ short offcodeNCount[MaxOff+1]; { short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; U32 offcodeMaxValue=MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; { short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; { short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
@@ -1474,7 +1534,8 @@ typedef enum { zdss_init, zdss_loadHeader,
/* *** Resource management *** */ /* *** Resource management *** */
struct ZSTD_DStream_s { struct ZSTD_DStream_s {
ZSTD_DCtx* dctx; ZSTD_DCtx* dctx;
ZSTD_DDict* ddict; ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
ZSTD_dStreamStage stage; ZSTD_dStreamStage stage;
char* inBuff; char* inBuff;
@@ -1524,7 +1585,7 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
if (zds==NULL) return 0; /* support free on null */ if (zds==NULL) return 0; /* support free on null */
{ ZSTD_customMem const cMem = zds->customMem; { ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->dctx); ZSTD_freeDCtx(zds->dctx);
ZSTD_freeDDict(zds->ddict); ZSTD_freeDDict(zds->ddictLocal);
ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->inBuff, cMem);
ZSTD_free(zds->outBuff, cMem); ZSTD_free(zds->outBuff, cMem);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
@@ -1546,9 +1607,12 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
{ {
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
ZSTD_freeDDict(zds->ddict); ZSTD_freeDDict(zds->ddictLocal);
zds->ddict = ZSTD_createDDict(dict, dictSize); if (dict) {
if (zds->ddict == NULL) return ERROR(memory_allocation); zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
} else zds->ddictLocal = NULL;
zds->ddict = zds->ddictLocal;
zds->legacyVersion = 0; zds->legacyVersion = 0;
zds->hostageByte = 0; zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix; return ZSTD_frameHeaderSize_prefix;
@@ -1559,9 +1623,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0); return ZSTD_initDStream_usingDict(zds, NULL, 0);
} }
size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */
{
size_t const initResult = ZSTD_initDStream(zds);
zds->ddict = ddict;
return initResult;
}
size_t ZSTD_resetDStream(ZSTD_DStream* zds) size_t ZSTD_resetDStream(ZSTD_DStream* zds)
{ {
if (zds->ddict == NULL) return ERROR(stage_wrong); /* must be init at least once */
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
zds->legacyVersion = 0; zds->legacyVersion = 0;
@@ -1584,7 +1654,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{ {
if (zds==NULL) return 0; /* support sizeof on NULL */ if (zds==NULL) return 0; /* support sizeof on NULL */
return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize; return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize;
} }
@@ -1625,15 +1695,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) { if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dict : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
zds->ddict->dict, zds->ddict->dictSize)); dict, dictSize));
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
} else { } else {
return hSize; /* error */ return hSize; /* error */
} } } }
#else #else
return hSize; return hSize;
#endif #endif
if (hSize != 0) { /* need more input */ if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */
@@ -1648,7 +1720,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} } } }
/* Consume header */ /* Consume header */
ZSTD_refDCtx(zds->dctx, zds->ddict->refContext); { const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL;
ZSTD_refDCtx(zds->dctx, refContext);
}
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
{ size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);

10
C/zstd/zstd_internal.h
View File

@@ -31,6 +31,16 @@
# endif /* __STDC_VERSION__ */ # endif /* __STDC_VERSION__ */
#endif #endif
#ifdef _MSC_VER
# define FORCE_NOINLINE static __declspec(noinline)
#else
# ifdef __GNUC__
# define FORCE_NOINLINE static __attribute__((__noinline__))
# else
# define FORCE_NOINLINE static
# endif
#endif
/*-************************************* /*-*************************************
* Dependencies * Dependencies

92
C/zstd/zstd_opt.h
View File

@@ -1,5 +1,5 @@
/** /**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc. * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved. * All rights reserved.
* *
* This source code is licensed under the BSD-style license found in the * This source code is licensed under the BSD-style license found in the
@@ -16,6 +16,7 @@
#define ZSTD_FREQ_DIV 5 #define ZSTD_FREQ_DIV 5
#define ZSTD_MAX_PRICE (1<<30)
/*-************************************* /*-*************************************
* Price functions for optimal parser * Price functions for optimal parser
@@ -120,12 +121,14 @@ FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BY
} }
FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra)
{ {
/* offset */ /* offset */
BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1);
U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); U32 price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1);
if (!ultra && offCode >= 20) price += (offCode-19)*2;
/* match Length */ /* match Length */
{ const BYTE ML_deltaCode = 36; { const BYTE ML_deltaCode = 36;
const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength];
@@ -171,7 +174,7 @@ MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const B
#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ #define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \
{ \ { \
while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \
opt[pos].mlen = mlen_; \ opt[pos].mlen = mlen_; \
opt[pos].off = offset_; \ opt[pos].off = offset_; \
opt[pos].litlen = litlen_; \ opt[pos].litlen = litlen_; \
@@ -375,7 +378,7 @@ static U32 ZSTD_BtGetAllMatches_selectMLS_extDict (
*********************************/ *********************************/
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize) const void* src, size_t srcSize, const int ultra)
{ {
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src; const BYTE* const istart = (const BYTE*)src;
@@ -401,7 +404,6 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_rescaleFreqs(seqStorePtr); ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart); ip += (ip==prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
//inr = ip;
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
@@ -424,7 +426,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
} }
best_off = i - (ip == anchor); best_off = i - (ip == anchor);
do { do {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--; mlen--;
@@ -449,7 +451,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
best_mlen = matches[u].len; best_mlen = matches[u].len;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */
mlen++; mlen++;
@@ -496,7 +498,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
} }
best_mlen = minMatch; best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */ for (i=(opt[cur].mlen != 1); i<last_i; i++) { /* check rep */
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
@@ -510,21 +512,20 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
} }
best_off = i - (opt[cur].mlen != 1); best_off = i - (opt[cur].mlen != 1);
if (mlen > best_mlen) best_mlen = mlen;
if (opt[cur].mlen == 1) { do {
litlen = opt[cur].litlen; if (opt[cur].mlen == 1) {
if (cur > litlen) { litlen = opt[cur].litlen;
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH); if (cur > litlen) {
} else price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); } else
} else { price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
litlen = 0; } else {
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH); litlen = 0;
} price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
}
if (mlen > best_mlen) best_mlen = mlen;
do {
if (cur + mlen > last_pos || price <= opt[cur + mlen].price) if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price); SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--; mlen--;
@@ -549,12 +550,12 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
else else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
} else { } else {
litlen = 0; litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
} }
if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) if (cur + mlen > last_pos || (price < opt[cur + mlen].price))
@@ -626,7 +627,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
FORCE_INLINE FORCE_INLINE
void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
const void* src, size_t srcSize) const void* src, size_t srcSize, const int ultra)
{ {
seqStore_t* seqStorePtr = &(ctx->seqStore); seqStore_t* seqStorePtr = &(ctx->seqStore);
const BYTE* const istart = (const BYTE*)src; const BYTE* const istart = (const BYTE*)src;
@@ -657,7 +658,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate; ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr); ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart); ip += (ip==prefixStart);
//inr = ip;
/* Match Loop */ /* Match Loop */
while (ip < ilimit) { while (ip < ilimit) {
@@ -666,7 +666,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base); U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t)); memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0; last_pos = 0;
//inr = ip;
opt[0].litlen = (U32)(ip - anchor); opt[0].litlen = (U32)(ip - anchor);
/* check repCode */ /* check repCode */
@@ -691,7 +690,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_off = i - (ip==anchor); best_off = i - (ip==anchor);
litlen = opt[0].litlen; litlen = opt[0].litlen;
do { do {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */
mlen--; mlen--;
@@ -721,7 +720,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
best_mlen = matches[u].len; best_mlen = matches[u].len;
litlen = opt[0].litlen; litlen = opt[0].litlen;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
if (mlen > last_pos || price < opt[mlen].price) if (mlen > last_pos || price < opt[mlen].price)
SET_PRICE(mlen, mlen, matches[u].off, litlen, price); SET_PRICE(mlen, mlen, matches[u].off, litlen, price);
mlen++; mlen++;
@@ -765,8 +764,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]);
} }
best_mlen = 0; best_mlen = minMatch;
{ U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1);
for (i = (mlen != 1); i<last_i; i++) { for (i = (mlen != 1); i<last_i; i++) {
const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i]; const S32 repCur = ((i==ZSTD_REP_MOVE_OPT) && (opt[cur].mlen != 1)) ? (opt[cur].rep[0] - 1) : opt[cur].rep[i];
@@ -786,20 +784,20 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
} }
best_off = i - (opt[cur].mlen != 1); best_off = i - (opt[cur].mlen != 1);
if (opt[cur].mlen == 1) { if (mlen > best_mlen) best_mlen = mlen;
litlen = opt[cur].litlen;
if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH);
} else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH);
}
best_mlen = mlen;
do { do {
if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen;
if (cur > litlen) {
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra);
} else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra);
} else {
litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra);
}
if (cur + mlen > last_pos || price <= opt[cur + mlen].price) if (cur + mlen > last_pos || price <= opt[cur + mlen].price)
SET_PRICE(cur + mlen, mlen, i, litlen, price); SET_PRICE(cur + mlen, mlen, i, litlen, price);
mlen--; mlen--;
@@ -815,8 +813,6 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
goto _storeSequence; goto _storeSequence;
} }
best_mlen = (best_mlen > minMatch) ? best_mlen : minMatch;
/* set prices using matches at position = cur */ /* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) { for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
@@ -826,12 +822,12 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
litlen = opt[cur].litlen; litlen = opt[cur].litlen;
if (cur > litlen) if (cur > litlen)
price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH); price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra);
else else
price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH); price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra);
} else { } else {
litlen = 0; litlen = 0;
price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH); price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra);
} }
if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) if (cur + mlen > last_pos || (price < opt[cur + mlen].price))

7
C/zstd/zstd_v01.c
View File

@@ -958,13 +958,16 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
U32 weightTotal; U32 weightTotal;
U32 maxBits; U32 maxBits;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
U32 n; U32 n;
U32 nextRankStart; U32 nextRankStart;
void* ptr = DTable+1; void* ptr = DTable+1;
HUF_DElt* const dt = (HUF_DElt*)ptr; HUF_DElt* const dt = (HUF_DElt*)ptr;
if (!srcSize) return (size_t)-FSE_ERROR_srcSize_wrong;
iSize = ip[0];
FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */ FSE_STATIC_ASSERT(sizeof(HUF_DElt) == sizeof(U16)); /* if compilation fails here, assertion is false */
//memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */ //memset(huffWeight, 0, sizeof(huffWeight)); /* should not be necessary, but some analyzer complain ... */
if (iSize >= 128) /* special header */ if (iSize >= 128) /* special header */
@@ -1005,6 +1008,7 @@ static size_t HUF_readDTable (U16* DTable, const void* src, size_t srcSize)
rankVal[huffWeight[n]]++; rankVal[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return (size_t)-FSE_ERROR_corruptionDetected;
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
maxBits = FSE_highbit32(weightTotal) + 1; maxBits = FSE_highbit32(weightTotal) + 1;
@@ -1533,6 +1537,7 @@ size_t ZSTDv01_decodeLiteralsBlock(void* ctx,
{ {
size_t rleSize = litbp.origSize; size_t rleSize = litbp.origSize;
if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall); if (rleSize>maxDstSize) return ERROR(dstSize_tooSmall);
if (!srcSize) return ERROR(srcSize_wrong);
memset(oend - rleSize, *ip, rleSize); memset(oend - rleSize, *ip, rleSize);
*litStart = oend - rleSize; *litStart = oend - rleSize;
*litSize = rleSize; *litSize = rleSize;

5
C/zstd/zstd_v02.c
View File

@@ -1607,10 +1607,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal; U32 weightTotal;
U32 tableLog; U32 tableLog;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
U32 n; U32 n;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */ if (iSize >= 128) /* special header */
@@ -1652,6 +1654,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1; tableLog = BIT_highbit32(weightTotal) + 1;

5
C/zstd/zstd_v03.c
View File

@@ -1604,10 +1604,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal; U32 weightTotal;
U32 tableLog; U32 tableLog;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
U32 n; U32 n;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */ if (iSize >= 128) /* special header */
@@ -1649,6 +1651,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1; tableLog = BIT_highbit32(weightTotal) + 1;

7
C/zstd/zstd_v04.c
View File

@@ -1896,10 +1896,12 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal; U32 weightTotal;
U32 tableLog; U32 tableLog;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
U32 n; U32 n;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) /* special header */ if (iSize >= 128) /* special header */
@@ -1941,6 +1943,7 @@ static size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BIT_highbit32(weightTotal) + 1; tableLog = BIT_highbit32(weightTotal) + 1;
@@ -3108,7 +3111,7 @@ static size_t ZSTD_execSequence(BYTE* op,
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_8) { if (op > oend_8) {
memmove(op, match, sequence.matchLength); while (op < oMatchEnd) *op++ = *match++;
return sequenceLength; return sequenceLength;
} }
} }

41
C/zstd/zstd_v05.c
View File

@@ -1873,10 +1873,12 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32 weightTotal; U32 weightTotal;
U32 tableLog; U32 tableLog;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
U32 n; U32 n;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */ if (iSize >= 128) { /* special header */
@@ -1910,6 +1912,7 @@ static size_t HUFv05_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
tableLog = BITv05_highbit32(weightTotal) + 1; tableLog = BITv05_highbit32(weightTotal) + 1;
@@ -2032,13 +2035,14 @@ size_t HUFv05_decompress1X2_usingDTable(
{ {
BYTE* op = (BYTE*)dst; BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize; BYTE* const oend = op + dstSize;
size_t errorCode;
const U32 dtLog = DTable[0]; const U32 dtLog = DTable[0];
const void* dtPtr = DTable; const void* dtPtr = DTable;
const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1; const HUFv05_DEltX2* const dt = ((const HUFv05_DEltX2*)dtPtr)+1;
BITv05_DStream_t bitD; BITv05_DStream_t bitD;
errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode; if (dstSize <= cSrcSize) return ERROR(dstSize_tooSmall);
{ size_t const errorCode = BITv05_initDStream(&bitD, cSrc, cSrcSize);
if (HUFv05_isError(errorCode)) return errorCode; }
HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog); HUFv05_decodeStreamX2(op, &bitD, oend, dt, dtLog);
@@ -2942,6 +2946,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
{ {
size_t litSize, litCSize, singleStream=0; size_t litSize, litCSize, singleStream=0;
U32 lhSize = ((istart[0]) >> 4) & 3; U32 lhSize = ((istart[0]) >> 4) & 3;
if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */
switch(lhSize) switch(lhSize)
{ {
case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */ case 0: case 1: default: /* note : default is impossible, since lhSize into [0..3] */
@@ -2965,6 +2970,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break; break;
} }
if (litSize > BLOCKSIZE) return ERROR(corruption_detected); if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUFv05_isError(singleStream ? if (HUFv05_isError(singleStream ?
HUFv05_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) : HUFv05_decompress1X2(dctx->litBuffer, litSize, istart+lhSize, litCSize) :
@@ -2990,6 +2996,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
lhSize=3; lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2]; litCSize = ((istart[1] & 3) << 8) + istart[2];
if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); errorCode = HUFv05_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUFv05_isError(errorCode)) return ERROR(corruption_detected); if (HUFv05_isError(errorCode)) return ERROR(corruption_detected);
@@ -3046,6 +3053,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
break; break;
case 3: case 3:
litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2]; litSize = ((istart[0] & 15) << 16) + (istart[1] << 8) + istart[2];
if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */
break; break;
} }
if (litSize > BLOCKSIZE) return ERROR(corruption_detected); if (litSize > BLOCKSIZE) return ERROR(corruption_detected);
@@ -3063,7 +3071,7 @@ size_t ZSTDv05_decodeLiteralsBlock(ZSTDv05_DCtx* dctx,
size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr, size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumpsLengthPtr,
FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb, FSEv05_DTable* DTableLL, FSEv05_DTable* DTableML, FSEv05_DTable* DTableOffb,
const void* src, size_t srcSize) const void* src, size_t srcSize, U32 flagStaticTable)
{ {
const BYTE* const istart = (const BYTE* const)src; const BYTE* const istart = (const BYTE* const)src;
const BYTE* ip = istart; const BYTE* ip = istart;
@@ -3079,17 +3087,22 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
/* SeqHead */ /* SeqHead */
*nbSeq = *ip++; *nbSeq = *ip++;
if (*nbSeq==0) return 1; if (*nbSeq==0) return 1;
if (*nbSeq >= 128) if (*nbSeq >= 128) {
if (ip >= iend) return ERROR(srcSize_wrong);
*nbSeq = ((nbSeq[0]-128)<<8) + *ip++; *nbSeq = ((nbSeq[0]-128)<<8) + *ip++;
}
if (ip >= iend) return ERROR(srcSize_wrong);
LLtype = *ip >> 6; LLtype = *ip >> 6;
Offtype = (*ip >> 4) & 3; Offtype = (*ip >> 4) & 3;
MLtype = (*ip >> 2) & 3; MLtype = (*ip >> 2) & 3;
if (*ip & 2) { if (*ip & 2) {
if (ip+3 > iend) return ERROR(srcSize_wrong);
dumpsLength = ip[2]; dumpsLength = ip[2];
dumpsLength += ip[1] << 8; dumpsLength += ip[1] << 8;
ip += 3; ip += 3;
} else { } else {
if (ip+2 > iend) return ERROR(srcSize_wrong);
dumpsLength = ip[1]; dumpsLength = ip[1];
dumpsLength += (ip[0] & 1) << 8; dumpsLength += (ip[0] & 1) << 8;
ip += 2; ip += 2;
@@ -3118,6 +3131,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableLL, LLbits); FSEv05_buildDTable_raw(DTableLL, LLbits);
break; break;
case FSEv05_ENCODING_STATIC: case FSEv05_ENCODING_STATIC:
if (!flagStaticTable) return ERROR(corruption_detected);
break; break;
case FSEv05_ENCODING_DYNAMIC : case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */ default : /* impossible */
@@ -3141,6 +3155,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableOffb, Offbits); FSEv05_buildDTable_raw(DTableOffb, Offbits);
break; break;
case FSEv05_ENCODING_STATIC: case FSEv05_ENCODING_STATIC:
if (!flagStaticTable) return ERROR(corruption_detected);
break; break;
case FSEv05_ENCODING_DYNAMIC : case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */ default : /* impossible */
@@ -3164,6 +3179,7 @@ size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t* dumps
FSEv05_buildDTable_raw(DTableML, MLbits); FSEv05_buildDTable_raw(DTableML, MLbits);
break; break;
case FSEv05_ENCODING_STATIC: case FSEv05_ENCODING_STATIC:
if (!flagStaticTable) return ERROR(corruption_detected);
break; break;
case FSEv05_ENCODING_DYNAMIC : case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */ default : /* impossible */
@@ -3313,7 +3329,7 @@ static size_t ZSTDv05_execSequence(BYTE* op,
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_8) { if (op > oend_8) {
memmove(op, match, sequence.matchLength); while (op < oMatchEnd) *op++ = *match++;
return sequenceLength; return sequenceLength;
} }
} } } }
@@ -3376,7 +3392,7 @@ static size_t ZSTDv05_decompressSequences(
/* Build Decoding Tables */ /* Build Decoding Tables */
errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength, errorCode = ZSTDv05_decodeSeqHeaders(&nbSeq, &dumps, &dumpsLength,
DTableLL, DTableML, DTableOffb, DTableLL, DTableML, DTableOffb,
ip, seqSize); ip, seqSize, dctx->flagStaticTables);
if (ZSTDv05_isError(errorCode)) return errorCode; if (ZSTDv05_isError(errorCode)) return errorCode;
ip += errorCode; ip += errorCode;
@@ -3665,11 +3681,11 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
{ {
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize; size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
short offcodeNCount[MaxOff+1]; short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSEv05Log; U32 offcodeMaxValue=MaxOff, offcodeLog;
short matchlengthNCount[MaxML+1]; short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSEv05Log; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
short litlengthNCount[MaxLL+1]; short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSEv05Log; unsigned litlengthMaxValue = MaxLL, litlengthLog;
hSize = HUFv05_readDTableX4(dctx->hufTableX4, dict, dictSize); hSize = HUFv05_readDTableX4(dctx->hufTableX4, dict, dictSize);
if (HUFv05_isError(hSize)) return ERROR(dictionary_corrupted); if (HUFv05_isError(hSize)) return ERROR(dictionary_corrupted);
@@ -3678,6 +3694,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
offcodeHeaderSize = FSEv05_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); offcodeHeaderSize = FSEv05_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSEv05_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv05_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSEv05Log) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); errorCode = FSEv05_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + offcodeHeaderSize; dict = (const char*)dict + offcodeHeaderSize;
@@ -3685,12 +3702,14 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
matchlengthHeaderSize = FSEv05_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); matchlengthHeaderSize = FSEv05_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSEv05_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv05_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSEv05Log) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); errorCode = FSEv05_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);
dict = (const char*)dict + matchlengthHeaderSize; dict = (const char*)dict + matchlengthHeaderSize;
dictSize -= matchlengthHeaderSize; dictSize -= matchlengthHeaderSize;
litlengthHeaderSize = FSEv05_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); litlengthHeaderSize = FSEv05_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (litlengthLog > LLFSEv05Log) return ERROR(dictionary_corrupted);
if (FSEv05_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv05_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
errorCode = FSEv05_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); errorCode = FSEv05_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted); if (FSEv05_isError(errorCode)) return ERROR(dictionary_corrupted);

24
C/zstd/zstd_v06.c
View File

@@ -1932,9 +1932,11 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
{ {
U32 weightTotal; U32 weightTotal;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */ if (iSize >= 128) { /* special header */
@@ -1969,6 +1971,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} } } }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv06_highbit32(weightTotal) + 1; { U32 const tableLog = BITv06_highbit32(weightTotal) + 1;
@@ -3183,6 +3186,7 @@ size_t ZSTDv06_decodeLiteralsBlock(ZSTDv06_DCtx* dctx,
lhSize=3; lhSize=3;
litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2); litSize = ((istart[0] & 15) << 6) + (istart[1] >> 2);
litCSize = ((istart[1] & 3) << 8) + istart[2]; litCSize = ((istart[1] & 3) << 8) + istart[2];
if (litCSize + litSize > srcSize) return ERROR(corruption_detected);
{ size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4); { size_t const errorCode = HUFv06_decompress1X4_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTableX4);
if (HUFv06_isError(errorCode)) return ERROR(corruption_detected); if (HUFv06_isError(errorCode)) return ERROR(corruption_detected);
@@ -3302,10 +3306,13 @@ size_t ZSTDv06_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++; { int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; } if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) { if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) if (nbSeq == 0xFF) {
if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
else } else {
if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++; nbSeq = ((nbSeq-0x80)<<8) + *ip++;
}
} }
*nbSeqPtr = nbSeq; *nbSeqPtr = nbSeq;
} }
@@ -3467,7 +3474,7 @@ size_t ZSTDv06_execSequence(BYTE* op,
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_8) { if (op > oend_8) {
memmove(op, match, sequence.matchLength); while (op < oMatchEnd) *op++ = *match++;
return sequenceLength; return sequenceLength;
} }
} } } }
@@ -3827,9 +3834,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
dictSize -= hSize; dictSize -= hSize;
{ short offcodeNCount[MaxOff+1]; { short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; U32 offcodeMaxValue=MaxOff, offcodeLog;
offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize); offcodeHeaderSize = FSEv06_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dict, dictSize);
if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv06_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); { size_t const errorCode = FSEv06_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + offcodeHeaderSize; dict = (const char*)dict + offcodeHeaderSize;
@@ -3837,9 +3845,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
} }
{ short matchlengthNCount[MaxML+1]; { short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize); matchlengthHeaderSize = FSEv06_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dict, dictSize);
if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv06_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); { size_t const errorCode = FSEv06_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
dict = (const char*)dict + matchlengthHeaderSize; dict = (const char*)dict + matchlengthHeaderSize;
@@ -3847,9 +3856,10 @@ static size_t ZSTDv06_loadEntropy(ZSTDv06_DCtx* dctx, const void* dict, size_t d
} }
{ short litlengthNCount[MaxLL+1]; { short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize); litlengthHeaderSize = FSEv06_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dict, dictSize);
if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv06_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); { size_t const errorCode = FSEv06_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSEv06_isError(errorCode)) return ERROR(dictionary_corrupted); }
} }

23
C/zstd/zstd_v07.c
View File

@@ -1382,9 +1382,11 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
{ {
U32 weightTotal; U32 weightTotal;
const BYTE* ip = (const BYTE*) src; const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0]; size_t iSize;
size_t oSize; size_t oSize;
if (!srcSize) return ERROR(srcSize_wrong);
iSize = ip[0];
//memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */ //memset(huffWeight, 0, hwSize); /* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */ if (iSize >= 128) { /* special header */
@@ -1419,6 +1421,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
rankStats[huffWeight[n]]++; rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1; weightTotal += (1 << huffWeight[n]) >> 1;
} } } }
if (weightTotal == 0) return ERROR(corruption_detected);
/* get last non-null symbol weight (implied, total must be 2^n) */ /* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BITv07_highbit32(weightTotal) + 1; { U32 const tableLog = BITv07_highbit32(weightTotal) + 1;
@@ -3529,10 +3532,13 @@ size_t ZSTDv07_decodeSeqHeaders(int* nbSeqPtr,
{ int nbSeq = *ip++; { int nbSeq = *ip++;
if (!nbSeq) { *nbSeqPtr=0; return 1; } if (!nbSeq) { *nbSeqPtr=0; return 1; }
if (nbSeq > 0x7F) { if (nbSeq > 0x7F) {
if (nbSeq == 0xFF) if (nbSeq == 0xFF) {
if (ip+2 > iend) return ERROR(srcSize_wrong);
nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2;
else } else {
if (ip >= iend) return ERROR(srcSize_wrong);
nbSeq = ((nbSeq-0x80)<<8) + *ip++; nbSeq = ((nbSeq-0x80)<<8) + *ip++;
}
} }
*nbSeqPtr = nbSeq; *nbSeqPtr = nbSeq;
} }
@@ -3691,7 +3697,7 @@ size_t ZSTDv07_execSequence(BYTE* op,
sequence.matchLength -= length1; sequence.matchLength -= length1;
match = base; match = base;
if (op > oend_w) { if (op > oend_w) {
memmove(op, match, sequence.matchLength); while (op < oMatchEnd) *op++ = *match++;
return sequenceLength; return sequenceLength;
} }
} } } }
@@ -4102,27 +4108,30 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si
} }
{ short offcodeNCount[MaxOff+1]; { short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; U32 offcodeMaxValue=MaxOff, offcodeLog;
size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSEv07_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv07_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); { size_t const errorCode = FSEv07_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
{ short matchlengthNCount[MaxML+1]; { short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog;
size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSEv07_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv07_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); { size_t const errorCode = FSEv07_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
{ short litlengthNCount[MaxLL+1]; { short litlengthNCount[MaxLL+1];
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog;
size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSEv07_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSEv07_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); { size_t const errorCode = FSEv07_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog);
if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); } if (FSEv07_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;