xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-10 08:07:09 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update to LZ4 v1.7.5

Browse Source
This commit is contained in:
Tino Reichardt
2017-02-06 20:15:11 +01:00
parent f4f2729f08
commit 2e656cea76
7 changed files with 450 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
C/lz4/lz4.c
View File

@@ -391,6 +391,7 @@ typedef enum { full = 0, partial = 1 } earlyEnd_directive;
* Local Utils * Local Utils
**************************************/ **************************************/
int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; }
const char* LZ4_versionString(void) { return LZ4_VERSION_STRING; }
int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); }
int LZ4_sizeofState() { return LZ4_STREAMSIZE; } int LZ4_sizeofState() { return LZ4_STREAMSIZE; }

2
C/lz4/lz4.h
View File

@@ -85,7 +85,7 @@ extern "C" {
/*========== Version =========== */ /*========== Version =========== */
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ #define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ #define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 4 /* for tweaks, bug-fixes, or development */ #define LZ4_VERSION_RELEASE 5 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) #define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)

29
C/lz4/lz4frame.c
View File

@@ -158,7 +158,7 @@ static void LZ4F_writeLE64 (void* dst, U64 value64)
#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB #define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB
static const size_t minFHSize = 7; static const size_t minFHSize = 7;
static const size_t maxFHSize = 15; /* max Frame Header Size */ static const size_t maxFHSize = LZ4F_HEADER_SIZE_MAX; /* 15 */
static const size_t BHSize = 4; static const size_t BHSize = 4;
@@ -254,20 +254,27 @@ static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSI
return requestedBSID; return requestedBSID;
} }
/* LZ4F_compressBound() :
* Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
* prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
* Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
* When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/
static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered) static size_t LZ4F_compressBound_internal(size_t srcSize, const LZ4F_preferences_t* preferencesPtr, size_t alreadyBuffered)
{ {
LZ4F_preferences_t prefsNull; LZ4F_preferences_t prefsNull;
memset(&prefsNull, 0, sizeof(prefsNull)); memset(&prefsNull, 0, sizeof(prefsNull));
prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */
{ const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; { const LZ4F_preferences_t* const prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr;
U32 const flush = prefsPtr->autoFlush | (srcSize==0);
LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID; LZ4F_blockSizeID_t const bid = prefsPtr->frameInfo.blockSizeID;
size_t const blockSize = LZ4F_getBlockSize(bid); size_t const blockSize = LZ4F_getBlockSize(bid);
size_t const maxBuffered = blockSize - 1; size_t const maxBuffered = blockSize - 1;
size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered); size_t const bufferedSize = MIN(alreadyBuffered, maxBuffered);
size_t const maxSrcSize = srcSize + bufferedSize; size_t const maxSrcSize = srcSize + bufferedSize;
unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize); unsigned const nbFullBlocks = (unsigned)(maxSrcSize / blockSize);
size_t const partialBlockSize = srcSize & (blockSize-1); size_t const partialBlockSize = (srcSize - (srcSize==0)) & (blockSize-1); /* 0 => -1 == MAX => blockSize-1 */
size_t const lastBlockSize = prefsPtr->autoFlush ? partialBlockSize : 0; size_t const lastBlockSize = flush ? partialBlockSize : 0;
unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0); unsigned const nbBlocks = nbFullBlocks + (lastBlockSize>0);
size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */ size_t const blockHeaderSize = 4; /* default, without block CRC option (which cannot be generated with current API) */
@@ -322,7 +329,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
if (prefs.frameInfo.contentSize != 0) if (prefs.frameInfo.contentSize != 0)
prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */
if (prefs.compressionLevel < LZ4HC_MIN_CLEVEL) { if (prefs.compressionLevel < LZ4HC_CLEVEL_MIN) {
cctxI.lz4CtxPtr = &lz4ctx; cctxI.lz4CtxPtr = &lz4ctx;
cctxI.lz4CtxLevel = 1; cctxI.lz4CtxLevel = 1;
} }
@@ -349,7 +356,7 @@ size_t LZ4F_compressFrame(void* dstBuffer, size_t dstCapacity, const void* srcBu
if (LZ4F_isError(tailSize)) return tailSize; if (LZ4F_isError(tailSize)) return tailSize;
dstPtr += tailSize; } dstPtr += tailSize; }
if (prefs.compressionLevel >= LZ4HC_MIN_CLEVEL) /* no allocation done with lz4 fast */ if (prefs.compressionLevel >= LZ4HC_CLEVEL_MIN) /* no allocation done with lz4 fast */
FREEMEM(cctxI.lz4CtxPtr); FREEMEM(cctxI.lz4CtxPtr);
return (dstPtr - dstStart); return (dstPtr - dstStart);
@@ -398,7 +405,7 @@ LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_comp
/*! LZ4F_compressBegin() : /*! LZ4F_compressBegin() :
* will write the frame header into dstBuffer. * will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. * dstBuffer must be large enough to accommodate a header (dstCapacity). Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
* @return : number of bytes written into dstBuffer for the header * @return : number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError()) * or an error code (can be tested using LZ4F_isError())
*/ */
@@ -417,10 +424,10 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
cctxPtr->prefs = *preferencesPtr; cctxPtr->prefs = *preferencesPtr;
/* ctx Management */ /* ctx Management */
{ U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ { U32 const tableID = (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */
if (cctxPtr->lz4CtxLevel < tableID) { if (cctxPtr->lz4CtxLevel < tableID) {
FREEMEM(cctxPtr->lz4CtxPtr); FREEMEM(cctxPtr->lz4CtxPtr);
if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); cctxPtr->lz4CtxPtr = (void*)LZ4_createStream();
else else
cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC();
@@ -445,7 +452,7 @@ size_t LZ4F_compressBegin(LZ4F_cctx* cctxPtr, void* dstBuffer, size_t dstCapacit
cctxPtr->tmpIn = cctxPtr->tmpBuff; cctxPtr->tmpIn = cctxPtr->tmpBuff;
cctxPtr->tmpInSize = 0; cctxPtr->tmpInSize = 0;
XXH32_reset(&(cctxPtr->xxh), 0); XXH32_reset(&(cctxPtr->xxh), 0);
if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr));
else else
LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel);
@@ -526,7 +533,7 @@ static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char
static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level)
{ {
if (level < LZ4HC_MIN_CLEVEL) { if (level < LZ4HC_CLEVEL_MIN) {
if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState;
return LZ4F_localLZ4_compress_limitedOutput_continue; return LZ4F_localLZ4_compress_limitedOutput_continue;
} }
@@ -536,7 +543,7 @@ static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int lev
static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr)
{ {
if (cctxPtr->prefs.compressionLevel < LZ4HC_MIN_CLEVEL) if (cctxPtr->prefs.compressionLevel < LZ4HC_CLEVEL_MIN)
return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB);
} }

23
C/lz4/lz4frame.h
View File

@@ -206,10 +206,11 @@ LZ4FLIB_API LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_cctx* cctx);
/* Compression */ /* Compression */
#define LZ4F_HEADER_SIZE_MAX 15
LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr); LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBegin() : /* LZ4F_compressBegin() :
* will write the frame header into dstBuffer. * will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header. Maximum header size is 15 bytes. * dstCapacity must be large enough to store the header. Maximum header size is LZ4F_HEADER_SIZE_MAX bytes.
* `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default. * `prefsPtr` is optional : you can provide NULL as argument, all preferences will then be set to default.
* @return : number of bytes written into dstBuffer for the header * @return : number of bytes written into dstBuffer for the header
* or an error code (which can be tested using LZ4F_isError()) * or an error code (which can be tested using LZ4F_isError())
@@ -217,20 +218,20 @@ LZ4FLIB_API size_t LZ4F_compressBegin(LZ4F_cctx* cctx, void* dstBuffer, size_t d
LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); LZ4FLIB_API size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBound() : /* LZ4F_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations. * Provides dstCapacity given a srcSize to guarantee operation success in worst case situations.
* Different preferences can produce different results. * prefsPtr is optional : you can provide NULL as argument, preferences will be set to cover worst case scenario.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. * Result is always the same for a srcSize and prefsPtr, so it can be trusted to size reusable buffers.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) * When srcSize==0, LZ4F_compressBound() provides an upper bound for LZ4F_flush() and LZ4F_compressEnd() operations.
*/ */
LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); LZ4FLIB_API size_t LZ4F_compressUpdate(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressUpdate() : /* LZ4F_compressUpdate() :
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* An important rule is that dstBuffer MUST be large enough (dstCapacity) to ensure compression completion even in worst case. * An important rule is that dstCapacity MUST be large enough to ensure operation success even in worst case situations.
* This value is provided by using LZ4F_compressBound(). * This value is provided by LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
* LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized. * LZ4F_compressUpdate() doesn't guarantee error recovery. When an error occurs, compression context must be freed or resized.
* `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. * `cOptPtr` is optional : NULL can be provided, in which case all options are set to default.
* @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). * @return : number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered).
* or an error code if it fails (which can be tested using LZ4F_isError()) * or an error code if it fails (which can be tested using LZ4F_isError())
*/ */
@@ -245,12 +246,12 @@ LZ4FLIB_API size_t LZ4F_flush(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapaci
* or an error code if it fails (which can be tested using LZ4F_isError()) * or an error code if it fails (which can be tested using LZ4F_isError())
*/ */
LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); LZ4FLIB_API size_t LZ4F_compressEnd(LZ4F_cctx* cctx, void* dstBuffer, size_t dstCapacity, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressEnd() : /* LZ4F_compressEnd() :
* To properly finish the compressed frame, invoke LZ4F_compressEnd(). * To properly finish an LZ4 frame, invoke LZ4F_compressEnd().
* It will flush whatever data remained within `cctx` (like LZ4_flush()) * It will flush whatever data remained within `cctx` (like LZ4_flush())
* and properly finalize the frame, with an endMark and a checksum. * and properly finalize the frame, with an endMark and a checksum.
* `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. * `cOptPtr` is optional : NULL can be provided, in which case all options will be set to default.
* @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) * @return : number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* or an error code if it fails (which can be tested using LZ4F_isError()) * or an error code if it fails (which can be tested using LZ4F_isError())
* A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task. * A successful call to LZ4F_compressEnd() makes `cctx` available again for another compression task.

54
C/lz4/lz4hc.c
View File

@@ -84,7 +84,7 @@
* Local Macros * Local Macros
**************************************/ **************************************/
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG)) #define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-LZ4HC_HASH_LOG))
/* #define DELTANEXTU16(p) chainTable[(p) & LZ4HC_MAXD_MASK] */ /* flexible, LZ4HC_MAXD dependent */ #define DELTANEXTMAXD(p) chainTable[(p) & LZ4HC_MAXD_MASK] /* flexible, LZ4HC_MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ #define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
@@ -305,14 +305,15 @@ FORCE_INLINE int LZ4HC_encodeSequence (
return 0; return 0;
} }
#include "lz4opt.h"
static int LZ4HC_compress_generic ( static int LZ4HC_compress_hashChain (
LZ4HC_CCtx_internal* const ctx, LZ4HC_CCtx_internal* const ctx,
const char* const source, const char* const source,
char* const dest, char* const dest,
int const inputSize, int const inputSize,
int const maxOutputSize, int const maxOutputSize,
int compressionLevel, unsigned maxNbAttempts,
limitedOutput_directive limit limitedOutput_directive limit
) )
{ {
@@ -325,7 +326,6 @@ static int LZ4HC_compress_generic (
BYTE* op = (BYTE*) dest; BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize; BYTE* const oend = op + maxOutputSize;
unsigned maxNbAttempts;
int ml, ml2, ml3, ml0; int ml, ml2, ml3, ml0;
const BYTE* ref = NULL; const BYTE* ref = NULL;
const BYTE* start2 = NULL; const BYTE* start2 = NULL;
@@ -336,9 +336,6 @@ static int LZ4HC_compress_generic (
const BYTE* ref0; const BYTE* ref0;
/* init */ /* init */
if (compressionLevel > LZ4HC_MAX_CLEVEL) compressionLevel = LZ4HC_MAX_CLEVEL;
if (compressionLevel < 1) compressionLevel = LZ4HC_DEFAULT_CLEVEL;
maxNbAttempts = 1 << (compressionLevel-1);
ctx->end += inputSize; ctx->end += inputSize;
ip++; ip++;
@@ -489,6 +486,37 @@ _Search3:
return (int) (((char*)op)-dest); return (int) (((char*)op)-dest);
} }
static int LZ4HC_getSearchNum(int compressionLevel)
{
switch (compressionLevel) {
default: return 0; /* unused */
case 11: return 128;
case 12: return 1<<10;
}
}
static int LZ4HC_compress_generic (
LZ4HC_CCtx_internal* const ctx,
const char* const source,
char* const dest,
int const inputSize,
int const maxOutputSize,
int compressionLevel,
limitedOutput_directive limit
)
{
if (compressionLevel < 1) compressionLevel = LZ4HC_CLEVEL_DEFAULT;
if (compressionLevel > 9) {
switch (compressionLevel) {
case 10: return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (16-1), limit);
case 11: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, 128, 0);
default:
case 12: ctx->searchNum = LZ4HC_getSearchNum(compressionLevel); return LZ4HC_compress_optimal(ctx, source, dest, inputSize, maxOutputSize, limit, LZ4_OPT_NUM, 1);
}
}
return LZ4HC_compress_hashChain(ctx, source, dest, inputSize, maxOutputSize, 1 << (compressionLevel-1), limit);
}
int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); } int LZ4_sizeofStateHC(void) { return sizeof(LZ4_streamHC_t); }
@@ -534,6 +562,7 @@ void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */ LZ4_STATIC_ASSERT(sizeof(LZ4HC_CCtx_internal) <= sizeof(size_t) * LZ4_STREAMHCSIZE_SIZET); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
LZ4_streamHCPtr->internal_donotuse.base = NULL; LZ4_streamHCPtr->internal_donotuse.base = NULL;
LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel; LZ4_streamHCPtr->internal_donotuse.compressionLevel = (unsigned)compressionLevel;
LZ4_streamHCPtr->internal_donotuse.searchNum = LZ4HC_getSearchNum(compressionLevel);
} }
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
@@ -544,8 +573,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
dictSize = 64 KB; dictSize = 64 KB;
} }
LZ4HC_init (ctxPtr, (const BYTE*)dictionary); LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
ctxPtr->end = (const BYTE*)dictionary + dictSize; ctxPtr->end = (const BYTE*)dictionary + dictSize;
if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
else
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3);
return dictSize; return dictSize;
} }
@@ -554,7 +586,11 @@ int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int
static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock) static void LZ4HC_setExternalDict(LZ4HC_CCtx_internal* ctxPtr, const BYTE* newBlock)
{ {
if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ if (ctxPtr->compressionLevel >= LZ4HC_CLEVEL_OPT_MIN)
LZ4HC_updateBinTree(ctxPtr, ctxPtr->end - MFLIMIT, ctxPtr->end - LASTLITERALS);
else
if (ctxPtr->end >= ctxPtr->base + 4) LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */ /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit; ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);

17
C/lz4/lz4hc.h
View File

@@ -44,9 +44,10 @@ extern "C" {
/* --- Useful constants --- */ /* --- Useful constants --- */
#define LZ4HC_MIN_CLEVEL 3 #define LZ4HC_CLEVEL_MIN 3
#define LZ4HC_DEFAULT_CLEVEL 9 #define LZ4HC_CLEVEL_DEFAULT 9
#define LZ4HC_MAX_CLEVEL 16 #define LZ4HC_CLEVEL_OPT_MIN 11
#define LZ4HC_CLEVEL_MAX 12
/*-************************************ /*-************************************
@@ -58,7 +59,7 @@ extern "C" {
* Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h") * Compression is guaranteed to succeed if `dstCapacity >= LZ4_compressBound(srcSize)` (see "lz4.h")
* Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h") * Max supported `srcSize` value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
* `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work. * `compressionLevel` : Recommended values are between 4 and 9, although any value between 1 and LZ4HC_MAX_CLEVEL will work.
* Values >LZ4HC_MAX_CLEVEL behave the same as 16. * Values >LZ4HC_MAX_CLEVEL behave the same as LZ4HC_MAX_CLEVEL.
* @return : the number of bytes written into 'dst' * @return : the number of bytes written into 'dst'
* or 0 if compression fails. * or 0 if compression fails.
*/ */
@@ -130,11 +131,11 @@ LZ4LIB_API int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, in
* They are exposed to allow static allocation of `LZ4_streamHC_t`. * They are exposed to allow static allocation of `LZ4_streamHC_t`.
* Using these definitions makes the code vulnerable to potential API break when upgrading LZ4 * Using these definitions makes the code vulnerable to potential API break when upgrading LZ4
**************************************/ **************************************/
#define LZ4HC_DICTIONARY_LOGSIZE 16 #define LZ4HC_DICTIONARY_LOGSIZE 17
#define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE) #define LZ4HC_MAXD (1<<LZ4HC_DICTIONARY_LOGSIZE)
#define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1) #define LZ4HC_MAXD_MASK (LZ4HC_MAXD - 1)
#define LZ4HC_HASH_LOG (LZ4HC_DICTIONARY_LOGSIZE-1) #define LZ4HC_HASH_LOG 15
#define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG) #define LZ4HC_HASHTABLESIZE (1 << LZ4HC_HASH_LOG)
#define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1) #define LZ4HC_HASH_MASK (LZ4HC_HASHTABLESIZE - 1)
@@ -153,6 +154,7 @@ typedef struct
uint32_t dictLimit; /* below that point, need extDict */ uint32_t dictLimit; /* below that point, need extDict */
uint32_t lowLimit; /* below that point, no more dict */ uint32_t lowLimit; /* below that point, no more dict */
uint32_t nextToUpdate; /* index from which to continue dictionary update */ uint32_t nextToUpdate; /* index from which to continue dictionary update */
uint32_t searchNum; /* only for optimal parser */
uint32_t compressionLevel; uint32_t compressionLevel;
} LZ4HC_CCtx_internal; } LZ4HC_CCtx_internal;
@@ -169,12 +171,13 @@ typedef struct
unsigned int dictLimit; /* below that point, need extDict */ unsigned int dictLimit; /* below that point, need extDict */
unsigned int lowLimit; /* below that point, no more dict */ unsigned int lowLimit; /* below that point, no more dict */
unsigned int nextToUpdate; /* index from which to continue dictionary update */ unsigned int nextToUpdate; /* index from which to continue dictionary update */
unsigned int searchNum; /* only for optimal parser */
unsigned int compressionLevel; unsigned int compressionLevel;
} LZ4HC_CCtx_internal; } LZ4HC_CCtx_internal;
#endif #endif
#define LZ4_STREAMHCSIZE 262192 #define LZ4_STREAMHCSIZE (4*LZ4HC_HASHTABLESIZE + 2*LZ4HC_MAXD + 56) /* 393268 */
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) #define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
union LZ4_streamHC_u { union LZ4_streamHC_u {
size_t table[LZ4_STREAMHCSIZE_SIZET]; size_t table[LZ4_STREAMHCSIZE_SIZET];

363
C/lz4/lz4opt.h Normal file
View File

@@ -0,0 +1,363 @@
/*
lz4opt.h - Optimal Mode of LZ4
Copyright (C) 2015-2016, Przemyslaw Skibinski <inikep@gmail.com>
Note : this file is intended to be included within lz4hc.c
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/lz4/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#define LZ4_OPT_NUM (1<<12)
typedef struct
{
int off;
int len;
} LZ4HC_match_t;
typedef struct
{
int price;
int off;
int mlen;
int litlen;
} LZ4HC_optimal_t;
/* price in bits */
FORCE_INLINE size_t LZ4HC_literalsPrice(size_t litlen)
{
size_t price = 8*litlen;
if (litlen >= (size_t)RUN_MASK) price+=8*(1+(litlen-RUN_MASK)/255);
return price;
}
/* requires mlen >= MINMATCH */
FORCE_INLINE size_t LZ4HC_sequencePrice(size_t litlen, size_t mlen)
{
size_t price = 16 + 8; /* 16-bit offset + token */
price += LZ4HC_literalsPrice(litlen);
mlen -= MINMATCH;
if (mlen >= (size_t)ML_MASK) price+=8*(1+(mlen-ML_MASK)/255);
return price;
}
/*-*************************************
* Binary Tree search
***************************************/
FORCE_INLINE int LZ4HC_BinTree_InsertAndGetAllMatches (
LZ4HC_CCtx_internal* ctx,
const BYTE* const ip,
const BYTE* const iHighLimit,
size_t best_mlen,
LZ4HC_match_t* matches,
int* matchNum)
{
U16* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const U32 dictLimit = ctx->dictLimit;
const U32 current = (U32)(ip - base);
const U32 lowLimit = (ctx->lowLimit + MAX_DISTANCE > current) ? ctx->lowLimit : current - (MAX_DISTANCE - 1);
const BYTE* const dictBase = ctx->dictBase;
const BYTE* match;
int nbAttempts = ctx->searchNum;
int mnum = 0;
U16 *ptr0, *ptr1, delta0, delta1;
U32 matchIndex;
size_t matchLength = 0;
U32* HashPos;
if (ip + MINMATCH > iHighLimit) return 1;
/* HC4 match finder */
HashPos = &HashTable[LZ4HC_hashPtr(ip)];
matchIndex = *HashPos;
*HashPos = current;
ptr0 = &DELTANEXTMAXD(current*2+1);
ptr1 = &DELTANEXTMAXD(current*2);
delta0 = delta1 = (U16)(current - matchIndex);
while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
matchLength = LZ4_count(ip, match, iHighLimit);
} else {
const BYTE* vLimit = ip + (dictLimit - matchIndex);
match = dictBase + matchIndex;
if (vLimit > iHighLimit) vLimit = iHighLimit;
matchLength = LZ4_count(ip, match, vLimit);
if ((ip+matchLength == vLimit) && (vLimit < iHighLimit))
matchLength += LZ4_count(ip+matchLength, base+dictLimit, iHighLimit);
}
if (matchLength > best_mlen) {
best_mlen = matchLength;
if (matches) {
if (matchIndex >= dictLimit)
matches[mnum].off = (int)(ip - match);
else
matches[mnum].off = (int)(ip - (base + matchIndex)); /* virtual matchpos */
matches[mnum].len = (int)matchLength;
mnum++;
}
if (best_mlen > LZ4_OPT_NUM) break;
}
if (ip+matchLength >= iHighLimit) /* equal : no way to know if inf or sup */
break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */
if (*(ip+matchLength) < *(match+matchLength)) {
*ptr0 = delta0;
ptr0 = &DELTANEXTMAXD(matchIndex*2);
if (*ptr0 == (U16)-1) break;
delta0 = *ptr0;
delta1 += delta0;
matchIndex -= delta0;
} else {
*ptr1 = delta1;
ptr1 = &DELTANEXTMAXD(matchIndex*2+1);
if (*ptr1 == (U16)-1) break;
delta1 = *ptr1;
delta0 += delta1;
matchIndex -= delta1;
}
}
*ptr0 = (U16)-1;
*ptr1 = (U16)-1;
if (matchNum) *matchNum = mnum;
/* if (best_mlen > 8) return best_mlen-8; */
if (!matchNum) return 1;
return 1;
}
FORCE_INLINE void LZ4HC_updateBinTree(LZ4HC_CCtx_internal* ctx, const BYTE* const ip, const BYTE* const iHighLimit)
{
const BYTE* const base = ctx->base;
const U32 target = (U32)(ip - base);
U32 idx = ctx->nextToUpdate;
while(idx < target)
idx += LZ4HC_BinTree_InsertAndGetAllMatches(ctx, base+idx, iHighLimit, 8, NULL, NULL);
}
/** Tree updater, providing best match */
FORCE_INLINE int LZ4HC_BinTree_GetAllMatches (
LZ4HC_CCtx_internal* ctx,
const BYTE* const ip, const BYTE* const iHighLimit,
size_t best_mlen, LZ4HC_match_t* matches, const int fullUpdate)
{
int mnum = 0;
if (ip < ctx->base + ctx->nextToUpdate) return 0; /* skipped area */
if (fullUpdate) LZ4HC_updateBinTree(ctx, ip, iHighLimit);
best_mlen = LZ4HC_BinTree_InsertAndGetAllMatches(ctx, ip, iHighLimit, best_mlen, matches, &mnum);
ctx->nextToUpdate = (U32)(ip - ctx->base + best_mlen);
return mnum;
}
#define SET_PRICE(pos, mlen, offset, litlen, price) \
{ \
while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \
opt[pos].mlen = (int)mlen; \
opt[pos].off = (int)offset; \
opt[pos].litlen = (int)litlen; \
opt[pos].price = (int)price; \
}
static int LZ4HC_compress_optimal (
LZ4HC_CCtx_internal* ctx,
const char* const source,
char* dest,
int inputSize,
int maxOutputSize,
limitedOutput_directive limit,
const size_t sufficient_len,
const int fullUpdate
)
{
LZ4HC_optimal_t opt[LZ4_OPT_NUM + 1];
LZ4HC_match_t matches[LZ4_OPT_NUM + 1];
const BYTE *inr = NULL;
size_t res, cur, cur2;
size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
/* init */
ctx->end += inputSize;
ip++;
/* Main Loop */
while (ip < mflimit) {
memset(opt, 0, sizeof(LZ4HC_optimal_t));
last_pos = 0;
llen = ip - anchor;
match_num = LZ4HC_BinTree_GetAllMatches(ctx, ip, matchlimit, MINMATCH-1, matches, fullUpdate);
if (!match_num) { ip++; continue; }
if ((size_t)matches[match_num-1].len > sufficient_len) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
cur = 0;
last_pos = 1;
goto encode;
}
/* set prices using matches at position = 0 */
for (i = 0; i < match_num; i++) {
mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
best_mlen = (matches[i].len < LZ4_OPT_NUM) ? matches[i].len : LZ4_OPT_NUM;
while (mlen <= best_mlen) {
litlen = 0;
price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
mlen++;
}
}
if (last_pos < MINMATCH) { ip++; continue; }
/* check further positions */
opt[0].mlen = opt[1].mlen = 1;
for (cur = 1; cur <= last_pos; cur++) {
inr = ip + cur;
if (opt[cur-1].mlen == 1) {
litlen = opt[cur-1].litlen + 1;
if (cur != litlen) {
price = opt[cur - litlen].price + LZ4HC_literalsPrice(litlen);
} else {
price = LZ4HC_literalsPrice(llen + litlen) - LZ4HC_literalsPrice(llen);
}
} else {
litlen = 1;
price = opt[cur - 1].price + LZ4HC_literalsPrice(litlen);
}
mlen = 1;
best_mlen = 0;
if (cur > last_pos || price < (size_t)opt[cur].price)
SET_PRICE(cur, mlen, best_mlen, litlen, price);
if (cur == last_pos || inr >= mflimit) break;
match_num = LZ4HC_BinTree_GetAllMatches(ctx, inr, matchlimit, MINMATCH-1, matches, fullUpdate);
if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
last_pos = cur + 1;
goto encode;
}
/* set prices using matches at position = cur */
for (i = 0; i < match_num; i++) {
mlen = (i>0) ? (size_t)matches[i-1].len+1 : MINMATCH;
cur2 = cur;
best_mlen = (cur2 + matches[i].len < LZ4_OPT_NUM) ? (size_t)matches[i].len : LZ4_OPT_NUM - cur2;
while (mlen <= best_mlen) {
if (opt[cur2].mlen == 1) {
litlen = opt[cur2].litlen;
if (cur2 != litlen)
price = opt[cur2 - litlen].price + LZ4HC_sequencePrice(litlen, mlen);
else
price = LZ4HC_sequencePrice(llen + litlen, mlen) - LZ4HC_literalsPrice(llen);
} else {
litlen = 0;
price = opt[cur2].price + LZ4HC_sequencePrice(litlen, mlen);
}
if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) { // || (((int)price == opt[cur2 + mlen].price) && (opt[cur2 + mlen-1].mlen == 1))) {
SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
}
mlen++;
}
}
} /* for (cur = 1; cur <= last_pos; cur++) */
best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off;
cur = last_pos - best_mlen;
encode: /* cur, last_pos, best_mlen, best_off have to be set */
opt[0].mlen = 1;
while (1) {
mlen = opt[cur].mlen;
offset = opt[cur].off;
opt[cur].mlen = (int)best_mlen;
opt[cur].off = (int)best_off;
best_mlen = mlen;
best_off = offset;
if (mlen > cur) break;
cur -= mlen;
}
cur = 0;
while (cur < last_pos) {
mlen = opt[cur].mlen;
if (mlen == 1) { ip++; cur++; continue; }
offset = opt[cur].off;
cur += mlen;
res = LZ4HC_encodeSequence(&ip, &op, &anchor, (int)mlen, ip - offset, limit, oend);
if (res) return 0;
}
}
/* Encode Last Literals */
{ int lastRun = (int)(iend - anchor);
if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
}
/* End */
return (int) ((char*)op-dest);
}