xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-13 00:11:34 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update to ZStandard v1.1.3

Browse Source
This commit is contained in:
Tino Reichardt
2017-02-06 20:30:48 +01:00
parent 2e656cea76
commit 4dd6b0a816
12 changed files with 239 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
C/zstd/mem.h
View File

@@ -39,7 +39,7 @@ extern "C" {
#endif #endif
/* code only tested on 32 and 64 bits systems */ /* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } #define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }

191
C/zstd/zbuff.h
View File

@@ -1,191 +0,0 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/* ***************************************************************
* NOTES/WARNINGS
*****************************************************************/
/* The streaming API defined here will soon be deprecated by the
* new one in 'zstd.h'; consider migrating towards newer streaming
* API. See 'lib/README.md'.
*****************************************************************/
#ifndef ZSTD_BUFFERED_H_23987
#define ZSTD_BUFFERED_H_23987
#if defined (__cplusplus)
extern "C" {
#endif
/* *************************************
* Dependencies
***************************************/
#include <stddef.h> /* size_t */
/* ***************************************************************
* Compiler specifics
*****************************************************************/
/* ZSTD_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL */
#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport)
#else
# define ZSTDLIB_API
#endif
/* *************************************
* Streaming functions
***************************************/
/* This is the easier "buffered" streaming API,
* using an internal buffer to lift all restrictions on user-provided buffers
* which can be any size, any place, for both input and output.
* ZBUFF and ZSTD are 100% interoperable,
* frames created by one can be decoded by the other one */
typedef struct ZBUFF_CCtx_s ZBUFF_CCtx;
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx(void);
ZSTDLIB_API size_t ZBUFF_freeCCtx(ZBUFF_CCtx* cctx);
ZSTDLIB_API size_t ZBUFF_compressInit(ZBUFF_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZBUFF_compressContinue(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr, const void* src, size_t* srcSizePtr);
ZSTDLIB_API size_t ZBUFF_compressFlush(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
ZSTDLIB_API size_t ZBUFF_compressEnd(ZBUFF_CCtx* cctx, void* dst, size_t* dstCapacityPtr);
/*-*************************************************
* Streaming compression - howto
*
* A ZBUFF_CCtx object is required to track streaming operation.
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
* ZBUFF_CCtx objects can be reused multiple times.
*
* Start by initializing ZBUF_CCtx.
* Use ZBUFF_compressInit() to start a new compression operation.
* Use ZBUFF_compressInitDictionary() for a compression which requires a dictionary.
*
* Use ZBUFF_compressContinue() repetitively to consume input stream.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written within *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present again remaining data.
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each call, so save its content if it matters or change @dst .
* @return : a hint to preferred nb of bytes to use as input for next function call (it's just a hint, to improve latency)
* or an error code, which can be tested using ZBUFF_isError().
*
* At any moment, it's possible to flush whatever data remains within buffer, using ZBUFF_compressFlush().
* The nb of bytes written into `dst` will be reported into *dstCapacityPtr.
* Note that the function cannot output more than *dstCapacityPtr,
* therefore, some content might still be left into internal buffer if *dstCapacityPtr is too small.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressEnd() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* The epilogue is required for decoders to consider a frame completed.
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
* In which case, call again ZBUFF_compressFlush() to complete the flush.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : _recommended buffer_ sizes (not compulsory) : ZBUFF_recommendedCInSize() / ZBUFF_recommendedCOutSize()
* input : ZBUFF_recommendedCInSize==128 KB block size is the internal unit, use this value to reduce intermediate stages (better latency)
* output : ZBUFF_recommendedCOutSize==ZSTD_compressBound(128 KB) + 3 + 3 : ensures it's always possible to write/flush/end a full block. Skip some buffering.
* By using both, it ensures that input will be entirely consumed, and output will always contain the result, reducing intermediate buffering.
* **************************************************/
typedef struct ZBUFF_DCtx_s ZBUFF_DCtx;
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx(void);
ZSTDLIB_API size_t ZBUFF_freeDCtx(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInit(ZBUFF_DCtx* dctx);
ZSTDLIB_API size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* dctx, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZBUFF_decompressContinue(ZBUFF_DCtx* dctx,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr);
/*-***************************************************************************
* Streaming decompression howto
*
* A ZBUFF_DCtx object is required to track streaming operations.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation,
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFF_DCtx objects can be re-init multiple times.
*
* Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of `dst` will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change `dst`.
* @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush,
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency),
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
* output : ZBUFF_recommendedDOutSize== 128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFF_recommendedDInSize == 128KB + 3;
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
/* *************************************
* Tool functions
***************************************/
ZSTDLIB_API unsigned ZBUFF_isError(size_t errorCode);
ZSTDLIB_API const char* ZBUFF_getErrorName(size_t errorCode);
/** Functions below provide recommended buffer sizes for Compression or Decompression operations.
* These sizes are just hints, they tend to offer better latency */
ZSTDLIB_API size_t ZBUFF_recommendedCInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedCOutSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDInSize(void);
ZSTDLIB_API size_t ZBUFF_recommendedDOutSize(void);
#ifdef ZBUFF_STATIC_LINKING_ONLY
/* ====================================================================================
* The definitions in this section are considered experimental.
* They should never be used in association with a dynamic library, as they may change in the future.
* They are provided for advanced usages.
* Use them only in association with static linking.
* ==================================================================================== */
/*--- Dependency ---*/
#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_customMem */
#include "zstd.h"
/*--- Custom memory allocator ---*/
/*! ZBUFF_createCCtx_advanced() :
* Create a ZBUFF compression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZBUFF_createDCtx_advanced() :
* Create a ZBUFF decompression context using external alloc and free functions */
ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem);
/*--- Advanced Streaming Initialization ---*/
ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize);
#endif /* ZBUFF_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_BUFFERED_H_23987 */

80
C/zstd/zstd.h
View File

@@ -20,13 +20,16 @@ extern "C" {
/* ===== ZSTDLIB_API : control library symbols visibility ===== */ /* ===== ZSTDLIB_API : control library symbols visibility ===== */
#if defined(__GNUC__) && (__GNUC__ >= 4) #if defined(__GNUC__) && (__GNUC__ >= 4)
# define ZSTDLIB_API __attribute__ ((visibility ("default"))) # define ZSTDLIB_VISIBILITY __attribute__ ((visibility ("default")))
#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport)
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else #else
# define ZSTDLIB_API # define ZSTDLIB_VISIBILITY
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBILITY
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define ZSTDLIB_API ZSTDLIB_VISIBILITY
#endif #endif
@@ -53,7 +56,7 @@ extern "C" {
/*------ Version ------*/ /*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1 #define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 2 #define ZSTD_VERSION_RELEASE 3
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str #define ZSTD_QUOTE(str) #str
@@ -170,8 +173,8 @@ typedef struct ZSTD_CDict_s ZSTD_CDict;
* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once.
* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay.
* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. * ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only.
* `dict` can be released after ZSTD_CDict creation. */ * `dictBuffer` can be released after ZSTD_CDict creation, as its content is copied within CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_freeCDict() : /*! ZSTD_freeCDict() :
* Function frees memory allocated by ZSTD_createCDict(). */ * Function frees memory allocated by ZSTD_createCDict(). */
@@ -191,8 +194,8 @@ typedef struct ZSTD_DDict_s ZSTD_DDict;
/*! ZSTD_createDDict() : /*! ZSTD_createDDict() :
* Create a digested dictionary, ready to start decompression operation without startup delay. * Create a digested dictionary, ready to start decompression operation without startup delay.
* `dict` can be released after creation. */ * dictBuffer can be released after DDict creation, as its content is copied inside DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize);
/*! ZSTD_freeDDict() : /*! ZSTD_freeDDict() :
* Function frees memory allocated with ZSTD_createDDict() */ * Function frees memory allocated with ZSTD_createDDict() */
@@ -325,7 +328,7 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
* ***************************************************************************************/ * ***************************************************************************************/
/* --- Constants ---*/ /* --- Constants ---*/
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ #define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX_32 25 #define ZSTD_WINDOWLOG_MAX_32 25
@@ -345,8 +348,9 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
#define ZSTD_FRAMEHEADERSIZE_MIN 6
static const size_t ZSTD_frameHeaderSize_prefix = 5; static const size_t ZSTD_frameHeaderSize_prefix = 5;
static const size_t ZSTD_frameHeaderSize_min = 6; static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
@@ -365,9 +369,9 @@ typedef struct {
} ZSTD_compressionParameters; } ZSTD_compressionParameters;
typedef struct { typedef struct {
unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */ unsigned contentSizeFlag; /**< 1: content size will be in frame header (when known) */
unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ unsigned checksumFlag; /**< 1: generate a 32-bits checksum at end of frame, for error detection */
unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */ unsigned noDictIDFlag; /**< 1: no dictID will be saved into frame header (if dictionary compression) */
} ZSTD_frameParameters; } ZSTD_frameParameters;
typedef struct { typedef struct {
@@ -397,9 +401,23 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
* Gives the amount of memory used by a given ZSTD_CCtx */ * Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
typedef enum {
ZSTD_p_forceWindow /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
} ZSTD_CCtxParameter;
/*! ZSTD_setCCtxParameter() :
* Set advanced parameters, selected through enum ZSTD_CCtxParameter
* @result : 0, or an error code (which can be tested with ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value);
/*! ZSTD_createCDict_byReference() :
* Create a digested dictionary for compression
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel);
/*! ZSTD_createCDict_advanced() : /*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem); ZSTD_parameters params, ZSTD_customMem customMem);
/*! ZSTD_sizeof_CDict() : /*! ZSTD_sizeof_CDict() :
@@ -455,6 +473,15 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
* Gives the amount of memory used by a given ZSTD_DCtx */ * Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
/*! ZSTD_sizeof_DDict() : /*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */ * Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
@@ -463,13 +490,13 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
* Provides the dictID stored within dictionary. * Provides the dictID stored within dictionary.
* if @return == 0, the dictionary is not conformant with Zstandard specification. * if @return == 0, the dictionary is not conformant with Zstandard specification.
* It can still be loaded, but as a content-only dictionary. */ * It can still be loaded, but as a content-only dictionary. */
unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize);
/*! ZSTD_getDictID_fromDDict() : /*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`. * Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
/*! ZSTD_getDictID_fromFrame() : /*! ZSTD_getDictID_fromFrame() :
* Provides the dictID required to decompressed the frame stored within `src`. * Provides the dictID required to decompressed the frame stored within `src`.
@@ -481,7 +508,7 @@ unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict);
* - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`).
* - This is not a Zstandard frame. * - This is not a Zstandard frame.
* When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */
unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/******************************************************************** /********************************************************************
@@ -491,7 +518,7 @@ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/ /*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */ ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
@@ -500,9 +527,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
/*===== Advanced Streaming decompression functions =====*/ /*===== Advanced Streaming decompression functions =====*/
typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
@@ -542,10 +569,10 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
In which case, it will "discard" the relevant memory section from its history. In which case, it will "discard" the relevant memory section from its history.
Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum.
It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame, It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame.
Without last block mark, frames will be considered unfinished (broken) by decoders. Without last block mark, frames will be considered unfinished (corrupted) by decoders.
You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. `ZSTD_CCtx` object can be re-used (ZSTD_compressBegin()) to compress some new frame.
*/ */
/*===== Buffer-less streaming compression functions =====*/ /*===== Buffer-less streaming compression functions =====*/
@@ -553,6 +580,7 @@ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);

4
C/zstd/zstd_common.c
View File

@@ -43,10 +43,6 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
* provides error code string from enum */ * provides error code string from enum */
const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); }
/* --- ZBUFF Error Management (deprecated) --- */
unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); }
const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); }
/*=************************************************************** /*=**************************************************************
* Custom allocator * Custom allocator

145
C/zstd/zstd_compress.c
View File

@@ -51,8 +51,7 @@ static void ZSTD_resetSeqStore(seqStore_t* ssPtr)
/*-************************************* /*-*************************************
* Context memory management * Context memory management
***************************************/ ***************************************/
struct ZSTD_CCtx_s struct ZSTD_CCtx_s {
{
const BYTE* nextSrc; /* next block here to continue on current prefix */ const BYTE* nextSrc; /* next block here to continue on current prefix */
const BYTE* base; /* All regular indexes relative to this position */ const BYTE* base; /* All regular indexes relative to this position */
const BYTE* dictBase; /* extDict indexes relative to this position */ const BYTE* dictBase; /* extDict indexes relative to this position */
@@ -61,10 +60,11 @@ struct ZSTD_CCtx_s
U32 nextToUpdate; /* index from which to continue dictionary update */ U32 nextToUpdate; /* index from which to continue dictionary update */
U32 nextToUpdate3; /* index from which to continue dictionary update */ U32 nextToUpdate3; /* index from which to continue dictionary update */
U32 hashLog3; /* dispatch table : larger == faster, more memory */ U32 hashLog3; /* dispatch table : larger == faster, more memory */
U32 loadedDictEnd; U32 loadedDictEnd; /* index of end of dictionary */
U32 forceWindow; /* force back-references to respect limit of 1<<wLog, even for dictionary */
ZSTD_compressionStage_e stage; ZSTD_compressionStage_e stage;
U32 rep[ZSTD_REP_NUM]; U32 rep[ZSTD_REP_NUM];
U32 savedRep[ZSTD_REP_NUM]; U32 repToConfirm[ZSTD_REP_NUM];
U32 dictID; U32 dictID;
ZSTD_parameters params; ZSTD_parameters params;
void* workSpace; void* workSpace;
@@ -101,7 +101,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL; if (!cctx) return NULL;
memset(cctx, 0, sizeof(ZSTD_CCtx)); memset(cctx, 0, sizeof(ZSTD_CCtx));
memcpy(&(cctx->customMem), &customMem, sizeof(customMem)); cctx->customMem = customMem;
return cctx; return cctx;
} }
@@ -119,6 +119,15 @@ size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
return sizeof(*cctx) + cctx->workSpaceSize; return sizeof(*cctx) + cctx->workSpaceSize;
} }
size_t ZSTD_setCCtxParameter(ZSTD_CCtx* cctx, ZSTD_CCtxParameter param, unsigned value)
{
switch(param)
{
case ZSTD_p_forceWindow : cctx->forceWindow = value>0; cctx->loadedDictEnd = 0; return 0;
default: return ERROR(parameter_unknown);
}
}
const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */
{ {
return &(ctx->seqStore); return &(ctx->seqStore);
@@ -318,6 +327,14 @@ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
} }
} }
/* ZSTD_invalidateRepCodes() :
* ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant;
* do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) {
int i;
for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = 0;
}
/*! ZSTD_copyCCtx() : /*! ZSTD_copyCCtx() :
* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. * Duplicate an existing context `srcCCtx` into another one `dstCCtx`.
@@ -735,12 +752,19 @@ _check_compressibility:
if ((size_t)(op-ostart) >= maxCSize) return 0; } if ((size_t)(op-ostart) >= maxCSize) return 0; }
/* confirm repcodes */ /* confirm repcodes */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->savedRep[i]; } { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = zc->repToConfirm[i]; }
return op - ostart; return op - ostart;
} }
#if 0 /* for debug */
# define STORESEQ_DEBUG
#include <stdio.h> /* fprintf */
U32 g_startDebug = 0;
const BYTE* g_start = NULL;
#endif
/*! ZSTD_storeSeq() : /*! ZSTD_storeSeq() :
Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. Store a sequence (literal length, literals, offset code and match length code) into seqStore_t.
`offsetCode` : distance to match, or 0 == repCode. `offsetCode` : distance to match, or 0 == repCode.
@@ -748,13 +772,14 @@ _check_compressibility:
*/ */
MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode)
{ {
#if 0 /* for debug */ #ifdef STORESEQ_DEBUG
static const BYTE* g_start = NULL; if (g_startDebug) {
const U32 pos = (U32)((const BYTE*)literals - g_start); const U32 pos = (U32)((const BYTE*)literals - g_start);
if (g_start==NULL) g_start = (const BYTE*)literals; if (g_start==NULL) g_start = (const BYTE*)literals;
//if ((pos > 1) && (pos < 50000)) if ((pos > 1895000) && (pos < 1895300))
printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", fprintf(stderr, "Cpos %6u :%5u literals & match %3u bytes at distance %6u \n",
pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode);
}
#endif #endif
/* copy Literals */ /* copy Literals */
ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); ZSTD_wildcopy(seqStorePtr->lit, literals, litLength);
@@ -1004,8 +1029,8 @@ void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx,
} } } } } }
/* save reps for next block */ /* save reps for next block */
cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -1119,7 +1144,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
} } } } } }
/* save reps for next block */ /* save reps for next block */
ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -1273,8 +1298,8 @@ void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx,
} } } } } }
/* save reps for next block */ /* save reps for next block */
cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; cctx->repToConfirm[0] = offset_1 ? offset_1 : offsetSaved;
cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; cctx->repToConfirm[1] = offset_2 ? offset_2 : offsetSaved;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -1423,7 +1448,7 @@ static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx,
} } } } } }
/* save reps for next block */ /* save reps for next block */
ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -1955,8 +1980,8 @@ _storeSequence:
} } } }
/* Save reps for next block */ /* Save reps for next block */
ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset; ctx->repToConfirm[0] = offset_1 ? offset_1 : savedOffset;
ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset; ctx->repToConfirm[1] = offset_2 ? offset_2 : savedOffset;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -2150,7 +2175,7 @@ _storeSequence:
} } } }
/* Save reps for next block */ /* Save reps for next block */
ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; ctx->repToConfirm[0] = offset_1; ctx->repToConfirm[1] = offset_2;
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -2409,12 +2434,14 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx,
cctx->nextSrc = ip + srcSize; cctx->nextSrc = ip + srcSize;
{ size_t const cSize = frame ? if (srcSize) {
size_t const cSize = frame ?
ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) :
ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
return cSize + fhSize; return cSize + fhSize;
} } else
return fhSize;
} }
@@ -2450,7 +2477,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t
zc->dictBase = zc->base; zc->dictBase = zc->base;
zc->base += ip - zc->nextSrc; zc->base += ip - zc->nextSrc;
zc->nextToUpdate = zc->dictLimit; zc->nextToUpdate = zc->dictLimit;
zc->loadedDictEnd = (U32)(iend - zc->base); zc->loadedDictEnd = zc->forceWindow ? 0 : (U32)(iend - zc->base);
zc->nextSrc = iend; zc->nextSrc = iend;
if (srcSize <= HASH_READ_SIZE) return 0; if (srcSize <= HASH_READ_SIZE) return 0;
@@ -2557,9 +2584,9 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] == 0 || cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] == 0 || cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] == 0 || cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12; dictPtr += 12;
{ U32 offcodeMax = MaxOff; { U32 offcodeMax = MaxOff;
@@ -2594,7 +2621,6 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
} }
} }
/*! ZSTD_compressBegin_internal() : /*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
@@ -2626,9 +2652,9 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di
} }
size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel)
{ {
return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel); return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel);
} }
@@ -2733,7 +2759,8 @@ size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcS
/* ===== Dictionary API ===== */ /* ===== Dictionary API ===== */
struct ZSTD_CDict_s { struct ZSTD_CDict_s {
void* dictContent; void* dictBuffer;
const void* dictContent;
size_t dictContentSize; size_t dictContentSize;
ZSTD_CCtx* refContext; ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ }; /* typedef'd tp ZSTD_CDict within "zstd.h" */
@@ -2741,39 +2768,45 @@ struct ZSTD_CDict_s {
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{ {
if (cdict==NULL) return 0; /* support sizeof on NULL */ if (cdict==NULL) return 0; /* support sizeof on NULL */
return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; return ZSTD_sizeof_CCtx(cdict->refContext) + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict);
} }
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, unsigned byReference,
ZSTD_parameters params, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); { ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem);
void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem);
if (!dictContent || !cdict || !cctx) { if (!cdict || !cctx) {
ZSTD_free(dictContent, customMem);
ZSTD_free(cdict, customMem); ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem); ZSTD_free(cctx, customMem);
return NULL; return NULL;
} }
if (dictSize) { if ((byReference) || (!dictBuffer) || (!dictSize)) {
memcpy(dictContent, dict, dictSize); cdict->dictBuffer = NULL;
cdict->dictContent = dictBuffer;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(cctx, customMem); ZSTD_free(cdict, customMem); return NULL; }
memcpy(internalBuffer, dictBuffer, dictSize);
cdict->dictBuffer = internalBuffer;
cdict->dictContent = internalBuffer;
} }
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, cdict->dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem); ZSTD_free(cdict->dictBuffer, customMem);
ZSTD_free(cdict, customMem);
ZSTD_free(cctx, customMem); ZSTD_free(cctx, customMem);
ZSTD_free(cdict, customMem);
return NULL; return NULL;
} } } }
cdict->dictContent = dictContent;
cdict->dictContentSize = dictSize;
cdict->refContext = cctx; cdict->refContext = cctx;
cdict->dictContentSize = dictSize;
return cdict; return cdict;
} }
} }
@@ -2783,7 +2816,15 @@ ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionL
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1; params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, params, allocator); return ZSTD_createCDict_advanced(dict, dictSize, 0, params, allocator);
}
ZSTD_CDict* ZSTD_createCDict_byReference(const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize);
params.fParams.contentSizeFlag = 1;
return ZSTD_createCDict_advanced(dict, dictSize, 1, params, allocator);
} }
size_t ZSTD_freeCDict(ZSTD_CDict* cdict) size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
@@ -2791,7 +2832,7 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
if (cdict==NULL) return 0; /* support free on NULL */ if (cdict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = cdict->refContext->customMem; { ZSTD_customMem const cMem = cdict->refContext->customMem;
ZSTD_freeCCtx(cdict->refContext); ZSTD_freeCCtx(cdict->refContext);
ZSTD_free(cdict->dictContent, cMem); ZSTD_free(cdict->dictBuffer, cMem);
ZSTD_free(cdict, cMem); ZSTD_free(cdict, cMem);
return 0; return 0;
} }
@@ -2801,7 +2842,7 @@ static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) {
return ZSTD_getParamsFromCCtx(cdict->refContext); return ZSTD_getParamsFromCCtx(cdict->refContext);
} }
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize)
{ {
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
@@ -2900,7 +2941,7 @@ size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSO
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{ {
if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */ if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once => can't reset */
if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize))
else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize));
@@ -2937,9 +2978,9 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (zcs->outBuff == NULL) return ERROR(memory_allocation); if (zcs->outBuff == NULL) return ERROR(memory_allocation);
} }
if (dict) { if (dict && dictSize >= 8) {
ZSTD_freeCDict(zcs->cdictLocal); ZSTD_freeCDict(zcs->cdictLocal);
zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, 0, params, zcs->customMem);
if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); if (zcs->cdictLocal == NULL) return ERROR(memory_allocation);
zcs->cdict = zcs->cdictLocal; zcs->cdict = zcs->cdictLocal;
} else zcs->cdict = NULL; } else zcs->cdict = NULL;
@@ -2956,6 +2997,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict)
ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict);
size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0); size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0);
zcs->cdict = cdict; zcs->cdict = cdict;
zcs->cctx->dictID = params.fParams.noDictIDFlag ? 0 : cdict->refContext->dictID;
return initError; return initError;
} }
@@ -2967,7 +3009,8 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t di
size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize)
{ {
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); ZSTD_parameters params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0);
if (pledgedSrcSize) params.fParams.contentSizeFlag = 1;
return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize);
} }

61
C/zstd/zstd_decompress.c
View File

@@ -1444,7 +1444,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
#endif #endif
ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); CHECK_F(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize));
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
} }
@@ -1671,9 +1671,9 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12; dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1; dctx->litEntropy = dctx->fseEntropy = 1;
@@ -1713,39 +1713,44 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t
/* ====== ZSTD_DDict ====== */ /* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s { struct ZSTD_DDict_s {
void* dict; void* dictBuffer;
const void* dictContent;
size_t dictSize; size_t dictSize;
ZSTD_DCtx* refContext; ZSTD_DCtx* refContext;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */ }; /* typedef'd to ZSTD_DDict within "zstd.h" */
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, unsigned byReference, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
void* const dictContent = ZSTD_malloc(dictSize, customMem);
ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem);
if (!dictContent || !ddict || !dctx) { if (!ddict || !dctx) {
ZSTD_free(dictContent, customMem);
ZSTD_free(ddict, customMem); ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem); ZSTD_free(dctx, customMem);
return NULL; return NULL;
} }
if (dictSize) { if ((byReference) || (!dict) || (!dictSize)) {
memcpy(dictContent, dict, dictSize); ddict->dictBuffer = NULL;
ddict->dictContent = dict;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, customMem);
if (!internalBuffer) { ZSTD_free(dctx, customMem); ZSTD_free(ddict, customMem); return NULL; }
memcpy(internalBuffer, dict, dictSize);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
} }
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, ddict->dictContent, dictSize);
if (ZSTD_isError(errorCode)) { if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem); ZSTD_free(ddict->dictBuffer, customMem);
ZSTD_free(ddict, customMem); ZSTD_free(ddict, customMem);
ZSTD_free(dctx, customMem); ZSTD_free(dctx, customMem);
return NULL; return NULL;
} } } }
ddict->dict = dictContent;
ddict->dictSize = dictSize; ddict->dictSize = dictSize;
ddict->refContext = dctx; ddict->refContext = dctx;
return ddict; return ddict;
@@ -1758,15 +1763,27 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{ {
ZSTD_customMem const allocator = { NULL, NULL, NULL }; ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, allocator); return ZSTD_createDDict_advanced(dict, dictSize, 0, allocator);
} }
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, ready to start decompression operation without startup delay.
* Dictionary content is simply referenced, and therefore stays in dictBuffer.
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, 1, allocator);
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict) size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{ {
if (ddict==NULL) return 0; /* support free on NULL */ if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->refContext->customMem; { ZSTD_customMem const cMem = ddict->refContext->customMem;
ZSTD_freeDCtx(ddict->refContext); ZSTD_freeDCtx(ddict->refContext);
ZSTD_free(ddict->dict, cMem); ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem); ZSTD_free(ddict, cMem);
return 0; return 0;
} }
@@ -1775,7 +1792,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{ {
if (ddict==NULL) return 0; /* support sizeof on NULL */ if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; return sizeof(*ddict) + ZSTD_sizeof_DCtx(ddict->refContext) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
} }
/*! ZSTD_getDictID_fromDict() : /*! ZSTD_getDictID_fromDict() :
@@ -1796,7 +1813,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize)
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{ {
if (ddict==NULL) return 0; if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize); return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
} }
/*! ZSTD_getDictID_fromFrame() : /*! ZSTD_getDictID_fromFrame() :
@@ -1827,7 +1844,7 @@ size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
const ZSTD_DDict* ddict) const ZSTD_DDict* ddict)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dictContent, ddict->dictSize);
#endif #endif
ZSTD_refDCtx(dctx, ddict->refContext); ZSTD_refDCtx(dctx, ddict->refContext);
ZSTD_checkContinuity(dctx, dst); ZSTD_checkContinuity(dctx, dst);
@@ -1919,7 +1936,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
ZSTD_freeDDict(zds->ddictLocal); ZSTD_freeDDict(zds->ddictLocal);
if (dict) { if (dict && dictSize >= 8) {
zds->ddictLocal = ZSTD_createDDict(dict, dictSize); zds->ddictLocal = ZSTD_createDDict(dict, dictSize);
if (zds->ddictLocal == NULL) return ERROR(memory_allocation); if (zds->ddictLocal == NULL) return ERROR(memory_allocation);
} else zds->ddictLocal = NULL; } else zds->ddictLocal = NULL;
@@ -1956,7 +1973,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
switch(paramType) switch(paramType)
{ {
default : return ERROR(parameter_unknown); default : return ERROR(parameter_unknown);
case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; case DStream_p_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
} }
return 0; return 0;
} }
@@ -2007,7 +2024,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) { if (legacyVersion) {
const void* const dict = zds->ddict ? zds->ddict->dict : NULL; const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL;
size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0;
CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
dict, dictSize)); dict, dictSize));

18
C/zstd/zstd_errors.h
View File

@@ -18,6 +18,20 @@ extern "C" {
#include <stddef.h> /* size_t */ #include <stddef.h> /* size_t */
/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */
#if defined(__GNUC__) && (__GNUC__ >= 4)
# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default")))
#else
# define ZSTDERRORLIB_VISIBILITY
#endif
#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1)
# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY
#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1)
# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY
#endif
/*-**************************************** /*-****************************************
* error codes list * error codes list
******************************************/ ******************************************/
@@ -49,8 +63,8 @@ typedef enum {
/*! ZSTD_getErrorCode() : /*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */ which can be used to compare directly with enum list published into "error_public.h" */
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
const char* ZSTD_getErrorString(ZSTD_ErrorCode code); ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus) #if defined (__cplusplus)

9
C/zstd/zstd_internal.h
View File

@@ -267,4 +267,13 @@ MEM_STATIC U32 ZSTD_highbit32(U32 val)
} }
/* hidden functions */
/* ZSTD_invalidateRepCodes() :
* ensures next compression will not use repcodes from previous block.
* Note : only works with regular variant;
* do not use with extDict variant ! */
void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx);
#endif /* ZSTD_CCOMMON_H_MODULE */ #endif /* ZSTD_CCOMMON_H_MODULE */

12
C/zstd/zstd_opt.h
View File

@@ -38,7 +38,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src
ssPtr->cachedLiterals = NULL; ssPtr->cachedLiterals = NULL;
ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; ssPtr->cachedPrice = ssPtr->cachedLitLength = 0;
ssPtr->staticPrices = 0; ssPtr->staticPrices = 0;
if (ssPtr->litLengthSum == 0) { if (ssPtr->litLengthSum == 0) {
if (srcSize <= 1024) ssPtr->staticPrices = 1; if (srcSize <= 1024) ssPtr->staticPrices = 1;
@@ -56,7 +56,7 @@ MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t src
for (u=0; u<=MaxLit; u++) { for (u=0; u<=MaxLit; u++) {
ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV);
ssPtr->litSum += ssPtr->litFreq[u]; ssPtr->litSum += ssPtr->litFreq[u];
} }
for (u=0; u<=MaxLL; u++) for (u=0; u<=MaxLL; u++)
ssPtr->litLengthFreq[u] = 1; ssPtr->litLengthFreq[u] = 1;
@@ -634,7 +634,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} } /* for (cur=0; cur < last_pos; ) */ } } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; } { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
/* Last Literals */ /* Last Literals */
{ size_t const lastLLSize = iend - anchor; { size_t const lastLLSize = iend - anchor;
@@ -825,7 +825,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch);
if (match_num > 0 && matches[match_num-1].len > sufficient_len) { if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) {
best_mlen = matches[match_num-1].len; best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off; best_off = matches[match_num-1].off;
last_pos = cur + 1; last_pos = cur + 1;
@@ -835,7 +835,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
/* set prices using matches at position = cur */ /* set prices using matches at position = cur */
for (u = 0; u < match_num; u++) { for (u = 0; u < match_num; u++) {
mlen = (u>0) ? matches[u-1].len+1 : best_mlen; mlen = (u>0) ? matches[u-1].len+1 : best_mlen;
best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; best_mlen = matches[u].len;
while (mlen <= best_mlen) { while (mlen <= best_mlen) {
if (opt[cur].mlen == 1) { if (opt[cur].mlen == 1) {
@@ -907,7 +907,7 @@ _storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */
} } /* for (cur=0; cur < last_pos; ) */ } } /* for (cur=0; cur < last_pos; ) */
/* Save reps for next block */ /* Save reps for next block */
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->savedRep[i] = rep[i]; } { int i; for (i=0; i<ZSTD_REP_NUM; i++) ctx->repToConfirm[i] = rep[i]; }
/* Last Literals */ /* Last Literals */
{ size_t lastLLSize = iend - anchor; { size_t lastLLSize = iend - anchor;

32
C/zstd/zstd_v04.c
View File

@@ -3012,21 +3012,19 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
/* Literal length */ /* Literal length */
litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream)); litLength = FSE_decodeSymbol(&(seqState->stateLL), &(seqState->DStream));
prevOffset = litLength ? seq->offset : seqState->prevOffset; prevOffset = litLength ? seq->offset : seqState->prevOffset;
if (litLength == MaxLL) if (litLength == MaxLL) {
{
U32 add = *dumps++; U32 add = *dumps++;
if (add < 255) litLength += add; if (add < 255) litLength += add;
else else {
{ litLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
litLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
dumps += 3; dumps += 3;
} }
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */ if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
} }
/* Offset */ /* Offset */
{ { static const U32 offsetPrefix[MaxOff+1] = {
static const U32 offsetPrefix[MaxOff+1] = {
1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256, 1 /*fake*/, 1, 2, 4, 8, 16, 32, 64, 128, 256,
512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 }; 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, /*fake*/ 1, 1, 1, 1, 1 };
@@ -3043,16 +3041,15 @@ static void ZSTD_decodeSequence(seq_t* seq, seqState_t* seqState)
/* MatchLength */ /* MatchLength */
matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream)); matchLength = FSE_decodeSymbol(&(seqState->stateML), &(seqState->DStream));
if (matchLength == MaxML) if (matchLength == MaxML) {
{
U32 add = *dumps++; U32 add = *dumps++;
if (add < 255) matchLength += add; if (add < 255) matchLength += add;
else else {
{ matchLength = dumps[0] + (dumps[1]<<8) + (dumps[2]<<16);
matchLength = MEM_readLE32(dumps) & 0xFFFFFF; /* no pb : dumps is always followed by seq tables > 1 byte */
dumps += 3; dumps += 3;
} }
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */ if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
} }
matchLength += MINMATCH; matchLength += MINMATCH;
@@ -3116,8 +3113,7 @@ static size_t ZSTD_execSequence(BYTE* op,
/* Requirement: op <= oend_8 */ /* Requirement: op <= oend_8 */
/* match within prefix */ /* match within prefix */
if (sequence.offset < 8) if (sequence.offset < 8) {
{
/* close range match, overlap */ /* close range match, overlap */
const int sub2 = dec64table[sequence.offset]; const int sub2 = dec64table[sequence.offset];
op[0] = match[0]; op[0] = match[0];
@@ -3127,9 +3123,7 @@ static size_t ZSTD_execSequence(BYTE* op,
match += dec32table[sequence.offset]; match += dec32table[sequence.offset];
ZSTD_copy4(op+4, match); ZSTD_copy4(op+4, match);
match -= sub2; match -= sub2;
} } else {
else
{
ZSTD_copy8(op, match); ZSTD_copy8(op, match);
} }
op += 8; match += 8; op += 8; match += 8;

6
C/zstd/zstd_v05.c
View File

@@ -3230,7 +3230,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
if (litLength&1) litLength>>=1, dumps += 3; if (litLength&1) litLength>>=1, dumps += 3;
else litLength = (U16)(litLength)>>1, dumps += 2; else litLength = (U16)(litLength)>>1, dumps += 2;
} }
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */ if (dumps > de) { litLength = MaxLL+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
} }
/* Offset */ /* Offset */
@@ -3263,7 +3264,8 @@ static void ZSTDv05_decodeSequence(seq_t* seq, seqState_t* seqState)
if (matchLength&1) matchLength>>=1, dumps += 3; if (matchLength&1) matchLength>>=1, dumps += 3;
else matchLength = (U16)(matchLength)>>1, dumps += 2; else matchLength = (U16)(matchLength)>>1, dumps += 2;
} }
if (dumps >= de) dumps = de-1; /* late correction, to avoid read overflow (data is now corrupted anyway) */ if (dumps > de) { matchLength = MaxML+255; } /* late correction, to avoid using uninitialized memory */
if (dumps >= de) { dumps = de-1; } /* late correction, to avoid read overflow (data is now corrupted anyway) */
} }
matchLength += MINMATCH; matchLength += MINMATCH;

6
C/zstd/zstd_v07.c
View File

@@ -4134,9 +4134,9 @@ static size_t ZSTDv07_loadEntropy(ZSTDv07_DCtx* dctx, const void* const dict, si
} }
if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted);
dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] == 0 || dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] == 0 || dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted);
dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] == 0 || dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted);
dictPtr += 12; dictPtr += 12;
dctx->litEntropy = dctx->fseEntropy = 1; dctx->litEntropy = dctx->fseEntropy = 1;