xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2026-01-31 12:24:08 -06:00
Code Issues Packages Projects Releases Wiki Activity

brotli + zstdmd update

Browse Source
This commit is contained in:
Tino Reichardt
2017-05-21 23:47:30 +02:00
parent 48ee1afd50
commit 294d7a00dc
144 changed files with 49097 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
C/lizard/bitstream.h
+414
View File
@@ -0,0 +1,414 @@
/* ******************************************************************
bitstream
Part of FSE library
header file (to include)
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef BITSTREAM_H_MODULE
#define BITSTREAM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*
* This API consists of small unitary functions, which must be inlined for best performance.
* Since link-time-optimization is not available for all compilers,
* these functions are defined into a .h to be included.
*/
/*-****************************************
* Dependencies
******************************************/
#include "mem.h" /* unaligned access routines */
#include "error_private.h" /* error codes and messages */
/*=========================================
* Target specific
=========================================*/
#if defined(__BMI__) && defined(__GNUC__)
# include <immintrin.h> /* support for bextr (experimental) */
#endif
/*-******************************************
* bitStream encoding API (write forward)
********************************************/
/* bitStream can mix input from multiple sources.
* A critical property of these streams is that they encode and decode in **reverse** direction.
* So the first bit sequence you add will be the last to be read, like a LIFO stack.
*/
typedef struct
{
size_t bitContainer;
int bitPos;
char* startPtr;
char* ptr;
char* endPtr;
} BIT_CStream_t;
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity);
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC);
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC);
/* Start with initCStream, providing the size of buffer to write into.
* bitStream will never write outside of this buffer.
* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code.
*
* bits are first added to a local register.
* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems.
* Writing data into memory is an explicit operation, performed by the flushBits function.
* Hence keep track how many bits are potentially stored into local register to avoid register overflow.
* After a flushBits, a maximum of 7 bits might still be stored into local register.
*
* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers.
*
* Last operation is to close the bitStream.
* The function returns the final size of CStream in bytes.
* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable)
*/
/*-********************************************
* bitStream decoding API (read backward)
**********************************************/
typedef struct
{
size_t bitContainer;
unsigned bitsConsumed;
const char* ptr;
const char* start;
} BIT_DStream_t;
typedef enum { BIT_DStream_unfinished = 0,
BIT_DStream_endOfBuffer = 1,
BIT_DStream_completed = 2,
BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */
/* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize);
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits);
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD);
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD);
/* Start by invoking BIT_initDStream().
* A chunk of the bitStream is then stored into a local register.
* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
* You can then retrieve bitFields stored into the local register, **in reverse order**.
* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method.
* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished.
* Otherwise, it can be less than that, so proceed accordingly.
* Checking if DStream has reached its end can be performed with BIT_endOfDStream().
*/
/*-****************************************
* unsafe API
******************************************/
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits);
/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC);
/* unsafe version; does not check buffer overflow */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
/* faster, but works only if nbBits >= 1 */
/*-**************************************************************
* Internal functions
****************************************************************/
MEM_STATIC unsigned BIT_highbit32 (register U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse ( &r, val );
return (unsigned) r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */
return 31 - __builtin_clz (val);
# else /* Software version */
static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
# endif
}
/*===== Local Constants =====*/
static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */
/*-**************************************************************
* bitStream encoding
****************************************************************/
/*! BIT_initCStream() :
* `dstCapacity` must be > sizeof(void*)
* @return : 0 if success,
otherwise an error code (can be tested using ERR_isError() ) */
MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity)
{
bitC->bitContainer = 0;
bitC->bitPos = 0;
bitC->startPtr = (char*)startPtr;
bitC->ptr = bitC->startPtr;
bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr);
if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall);
return 0;
}
/*! BIT_addBits() :
can add up to 26 bits into `bitC`.
Does not check for register overflow ! */
MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
{
bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_addBitsFast() :
* works only if `value` is _clean_, meaning all high bits above nbBits are 0 */
MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits)
{
bitC->bitContainer |= value << bitC->bitPos;
bitC->bitPos += nbBits;
}
/*! BIT_flushBitsFast() :
* unsafe version; does not check buffer overflow */
MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
{
size_t const nbBytes = bitC->bitPos >> 3;
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
}
/*! BIT_flushBits() :
* safe version; check for buffer overflow, and prevents it.
* note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */
MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
{
size_t const nbBytes = bitC->bitPos >> 3;
MEM_writeLEST(bitC->ptr, bitC->bitContainer);
bitC->ptr += nbBytes;
if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
bitC->bitPos &= 7;
bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */
}
/*! BIT_closeCStream() :
* @return : size of CStream, in bytes,
or 0 if it could not fit into dstBuffer */
MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC)
{
BIT_addBitsFast(bitC, 1, 1); /* endMark */
BIT_flushBits(bitC);
if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */
return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0);
}
/*-********************************************************
* bitStream decoding
**********************************************************/
/*! BIT_initDStream() :
* Initialize a BIT_DStream_t.
* `bitD` : a pointer to an already allocated BIT_DStream_t structure.
* `srcSize` must be the *exact* size of the bitStream, in bytes.
* @return : size of stream (== srcSize) or an errorCode if a problem is detected
*/
MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize)
{
if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); }
if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */
bitD->start = (const char*)srcBuffer;
bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer);
bitD->bitContainer = MEM_readLEST(bitD->ptr);
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
} else {
bitD->start = (const char*)srcBuffer;
bitD->ptr = bitD->start;
bitD->bitContainer = *(const BYTE*)(bitD->start);
switch(srcSize)
{
case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16);
case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24);
case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32);
case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24;
case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16;
case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8;
default:;
}
{ BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1];
bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0;
if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ }
bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8;
}
return srcSize;
}
MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start)
{
return bitContainer >> start;
}
MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental */
# if defined(__x86_64__)
if (sizeof(bitContainer)==8)
return _bextr_u64(bitContainer, start, nbBits);
else
# endif
return _bextr_u32(bitContainer, start, nbBits);
#else
return (bitContainer >> start) & BIT_mask[nbBits];
#endif
}
MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
{
return bitContainer & BIT_mask[nbBits];
}
/*! BIT_lookBits() :
* Provides next n bits from local register.
* local register is not modified.
* On 32-bits, maxNbBits==24.
* On 64-bits, maxNbBits==56.
* @return : value extracted
*/
MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits)
{
#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */
return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits);
#else
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask);
#endif
}
/*! BIT_lookBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits)
{
U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1;
return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask);
}
MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
{
bitD->bitsConsumed += nbBits;
}
/*! BIT_readBits() :
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value.
*/
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
{
size_t const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
{
size_t const value = BIT_lookBitsFast(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
return value;
}
/*! BIT_reloadDStream() :
* Refill `BIT_DStream_t` from src buffer previously defined (see BIT_initDStream() ).
* This function is safe, it guarantees it will not read beyond src buffer.
* @return : status of `BIT_DStream_t` internal register.
if status == unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */
MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD)
{
if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */
return BIT_DStream_overflow;
if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) {
bitD->ptr -= bitD->bitsConsumed >> 3;
bitD->bitsConsumed &= 7;
bitD->bitContainer = MEM_readLEST(bitD->ptr);
return BIT_DStream_unfinished;
}
if (bitD->ptr == bitD->start) {
if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer;
return BIT_DStream_completed;
}
{ U32 nbBytes = bitD->bitsConsumed >> 3;
BIT_DStream_status result = BIT_DStream_unfinished;
if (bitD->ptr - nbBytes < bitD->start) {
nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */
result = BIT_DStream_endOfBuffer;
}
bitD->ptr -= nbBytes;
bitD->bitsConsumed -= nbBytes*8;
bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */
return result;
}
}
/*! BIT_endOfDStream() :
* @return Tells if DStream has exactly reached its end (all bits consumed).
*/
MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream)
{
return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8));
}
#if defined (__cplusplus)
}
#endif
#endif /* BITSTREAM_H_MODULE */
C/lizard/entropy_common.c
+222
View File
@@ -0,0 +1,222 @@
/*
Common functions of New Generation Entropy library
Copyright (C) 2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
*************************************************************************** */
/* *************************************
* Dependencies
***************************************/
#include "mem.h"
#include "error_private.h" /* ERR_*, ERROR */
#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */
#include "huf.h"
/*-****************************************
* FSE Error Management
******************************************/
unsigned FSE_isError(size_t code) { return ERR_isError(code); }
const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); }
/* **************************************************************
* HUF Error Management
****************************************************************/
unsigned HUF_isError(size_t code) { return ERR_isError(code); }
const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr,
const void* headerBuffer, size_t hbSize)
{
const BYTE* const istart = (const BYTE*) headerBuffer;
const BYTE* const iend = istart + hbSize;
const BYTE* ip = istart;
int nbBits;
int remaining;
int threshold;
U32 bitStream;
int bitCount;
unsigned charnum = 0;
int previous0 = 0;
if (hbSize < 4) return ERROR(srcSize_wrong);
bitStream = MEM_readLE32(ip);
nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */
if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge);
bitStream >>= 4;
bitCount = 4;
*tableLogPtr = nbBits;
remaining = (1<<nbBits)+1;
threshold = 1<<nbBits;
nbBits++;
while ((remaining>1) & (charnum<=*maxSVPtr)) {
if (previous0) {
unsigned n0 = charnum;
while ((bitStream & 0xFFFF) == 0xFFFF) {
n0 += 24;
if (ip < iend-5) {
ip += 2;
bitStream = MEM_readLE32(ip) >> bitCount;
} else {
bitStream >>= 16;
bitCount += 16;
} }
while ((bitStream & 3) == 3) {
n0 += 3;
bitStream >>= 2;
bitCount += 2;
}
n0 += bitStream & 3;
bitCount += 2;
if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall);
while (charnum < n0) normalizedCounter[charnum++] = 0;
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
ip += bitCount>>3;
bitCount &= 7;
bitStream = MEM_readLE32(ip) >> bitCount;
} else {
bitStream >>= 2;
} }
{ short const max = (short)((2*threshold-1)-remaining);
short count;
if ((bitStream & (threshold-1)) < (U32)max) {
count = (short)(bitStream & (threshold-1));
bitCount += nbBits-1;
} else {
count = (short)(bitStream & (2*threshold-1));
if (count >= threshold) count -= max;
bitCount += nbBits;
}
count--; /* extra accuracy */
remaining -= FSE_abs(count);
normalizedCounter[charnum++] = count;
previous0 = !count;
while (remaining < threshold) {
nbBits--;
threshold >>= 1;
}
if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) {
ip += bitCount>>3;
bitCount &= 7;
} else {
bitCount -= (int)(8 * (iend - 4 - ip));
ip = iend - 4;
}
bitStream = MEM_readLE32(ip) >> (bitCount & 31);
} } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */
if (remaining != 1) return ERROR(corruption_detected);
if (bitCount > 32) return ERROR(corruption_detected);
*maxSVPtr = charnum-1;
ip += (bitCount+7)>>3;
return ip-istart;
}
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableX?() .
*/
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize)
{
U32 weightTotal;
const BYTE* ip = (const BYTE*) src;
size_t iSize = ip[0];
size_t oSize;
/* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */
if (iSize >= 128) { /* special header */
oSize = iSize - 127;
iSize = ((oSize+1)/2);
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
if (oSize >= hwSize) return ERROR(corruption_detected);
ip += 1;
{ U32 n;
for (n=0; n<oSize; n+=2) {
huffWeight[n] = ip[n/2] >> 4;
huffWeight[n+1] = ip[n/2] & 15;
} } }
else { /* header compressed with FSE (normal case) */
if (iSize+1 > srcSize) return ERROR(srcSize_wrong);
oSize = FSE_decompress(huffWeight, hwSize-1, ip+1, iSize); /* max (hwSize-1) values decoded, as last one is implied */
if (FSE_isError(oSize)) return oSize;
}
/* collect weight stats */
memset(rankStats, 0, (HUF_TABLELOG_ABSOLUTEMAX + 1) * sizeof(U32));
weightTotal = 0;
{ U32 n; for (n=0; n<oSize; n++) {
if (huffWeight[n] >= HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
rankStats[huffWeight[n]]++;
weightTotal += (1 << huffWeight[n]) >> 1;
} }
/* get last non-null symbol weight (implied, total must be 2^n) */
{ U32 const tableLog = BIT_highbit32(weightTotal) + 1;
if (tableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(corruption_detected);
*tableLogPtr = tableLog;
/* determine last weight */
{ U32 const total = 1 << tableLog;
U32 const rest = total - weightTotal;
U32 const verif = 1 << BIT_highbit32(rest);
U32 const lastWeight = BIT_highbit32(rest) + 1;
if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */
huffWeight[oSize] = (BYTE)lastWeight;
rankStats[lastWeight]++;
} }
/* check tree construction validity */
if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */
/* results */
*nbSymbolsPtr = (U32)(oSize+1);
return iSize+1;
}
C/lizard/error_private.h
+125
View File
@@ -0,0 +1,125 @@
/* ******************************************************************
Error codes and messages
Copyright (C) 2013-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
/* Note : this module is expected to remain private, do not expose it */
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t */
#include "error_public.h" /* enum list */
/* ****************************************
* Compiler-specific
******************************************/
#if defined(__GNUC__)
# define ERR_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
# define ERR_STATIC static __inline
#else
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Customization (error_public.h)
******************************************/
typedef ZSTD_ErrorCode ERR_enum;
#define PREFIX(name) ZSTD_error_##name
/*-****************************************
* Error codes handling
******************************************/
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define ERROR(name) ((size_t)-PREFIX(name))
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
/*-****************************************
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
{
static const char* notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
}
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */
C/lizard/error_public.h
+77
View File
@@ -0,0 +1,77 @@
/* ******************************************************************
Error codes list
Copyright (C) 2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* error codes list
******************************************/
typedef enum {
ZSTD_error_no_error,
ZSTD_error_GENERIC,
ZSTD_error_prefix_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
ZSTD_error_stage_wrong,
ZSTD_error_dstSize_tooSmall,
ZSTD_error_srcSize_wrong,
ZSTD_error_corruption_detected,
ZSTD_error_checksum_wrong,
ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,
ZSTD_error_dictionary_corrupted,
ZSTD_error_dictionary_wrong,
ZSTD_error_maxCode
} ZSTD_ErrorCode;
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */
C/lizard/fse.h
+628
View File
@@ -0,0 +1,628 @@
/* ******************************************************************
FSE : Finite State Entropy codec
Public Prototypes declaration
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef FSE_H
#define FSE_H
#if defined (__cplusplus)
extern "C" {
#endif
/*-*****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
/*-****************************************
* FSE simple functions
******************************************/
/*! FSE_compress() :
Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'.
'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize).
@return : size of compressed data (<= dstCapacity).
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead.
if FSE_isError(return), compression failed (more details using FSE_getErrorName())
*/
size_t FSE_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/*! FSE_decompress():
Decompress FSE data from buffer 'cSrc', of size 'cSrcSize',
into already allocated destination buffer 'dst', of size 'dstCapacity'.
@return : size of regenerated data (<= maxDstSize),
or an error code, which can be tested using FSE_isError() .
** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!!
Why ? : making this distinction requires a header.
Header management is intentionally delegated to the user layer, which can better manage special cases.
*/
size_t FSE_decompress(void* dst, size_t dstCapacity,
const void* cSrc, size_t cSrcSize);
/*-*****************************************
* Tool functions
******************************************/
size_t FSE_compressBound(size_t size); /* maximum compressed size */
/* Error Management */
unsigned FSE_isError(size_t code); /* tells if a return value is an error code */
const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */
/*-*****************************************
* FSE advanced functions
******************************************/
/*! FSE_compress2() :
Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog'
Both parameters can be defined as '0' to mean : use default value
@return : size of compressed data
Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!!
if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression.
if FSE_isError(return), it's an error code.
*/
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
/*-*****************************************
* FSE detailed API
******************************************/
/*!
FSE_compress() does the following:
1. count symbol occurrence from source[] into table count[]
2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog)
3. save normalized counters to memory buffer using writeNCount()
4. build encoding table 'CTable' from normalized counters
5. encode the data stream using encoding table 'CTable'
FSE_decompress() does the following:
1. read normalized counters with readNCount()
2. build decoding table 'DTable' from normalized counters
3. decode the data stream using decoding table 'DTable'
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and provide normalized distribution using external method.
*/
/* *** COMPRESSION *** */
/*! FSE_count():
Provides the precise count of each byte within a table 'count'.
'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1).
*maxSymbolValuePtr will be updated if detected smaller than initial value.
@return : the count of the most frequent symbol (which is not identified).
if return == srcSize, there is only one symbol.
Can also return an error code, which can be tested with FSE_isError(). */
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/*! FSE_optimalTableLog():
dynamically downsize 'tableLog' when conditions are met.
It saves CPU time, by using smaller tables, while preserving or even improving compression ratio.
@return : recommended tableLog (necessarily <= 'maxTableLog') */
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_normalizeCount():
normalize counts so that sum(count[]) == Power_of_2 (2^tableLog)
'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1).
@return : tableLog,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue);
/*! FSE_NCountWriteBound():
Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'.
Typically useful for allocation purpose. */
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_writeNCount():
Compactly save 'normalizedCounter' into 'buffer'.
@return : size of the compressed table,
or an errorCode, which can be tested using FSE_isError(). */
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! Constructor and Destructor of FSE_CTable.
Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */
typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */
FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue);
void FSE_freeCTable (FSE_CTable* ct);
/*! FSE_buildCTable():
Builds `ct`, which must be already allocated, using FSE_createCTable().
@return : 0, or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_compress_usingCTable():
Compress `src` using `ct` into `dst` which must be already allocated.
@return : size of compressed data (<= `dstCapacity`),
or 0 if compressed data could not fit into `dst`,
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct);
/*!
Tutorial :
----------
The first step is to count all symbols. FSE_count() does this job very fast.
Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells.
'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0]
maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value)
FSE_count() will return the number of occurrence of the most frequent symbol.
This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
The next step is to normalize the frequencies.
FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'.
It also guarantees a minimum of 1 to any Symbol with frequency >= 1.
You can use 'tableLog'==0 to mean "use default tableLog value".
If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(),
which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default").
The result of FSE_normalizeCount() will be saved into a table,
called 'normalizedCounter', which is a table of signed short.
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells.
The return value is tableLog if everything proceeded as expected.
It is 0 if there is a single symbol within distribution.
If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()).
'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount().
'buffer' must be already allocated.
For guaranteed success, buffer size must be at least FSE_headerBound().
The result of the function is the number of bytes written into 'buffer'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small).
'normalizedCounter' can then be used to create the compression table 'CTable'.
The space required by 'CTable' must be already allocated, using FSE_createCTable().
You can then use FSE_buildCTable() to fill 'CTable'.
If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()).
'CTable' can then be used to compress 'src', with FSE_compress_usingCTable().
Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize'
The function returns the size of compressed data (without header), necessarily <= `dstCapacity`.
If it returns '0', compressed data could not fit into 'dst'.
If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()).
*/
/* *** DECOMPRESSION *** */
/*! FSE_readNCount():
Read compactly saved 'normalizedCounter' from 'rBuffer'.
@return : size read from 'rBuffer',
or an errorCode, which can be tested using FSE_isError().
maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */
size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize);
/*! Constructor and Destructor of FSE_DTable.
Note that its size depends on 'tableLog' */
typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */
FSE_DTable* FSE_createDTable(unsigned tableLog);
void FSE_freeDTable(FSE_DTable* dt);
/*! FSE_buildDTable():
Builds 'dt', which must be already allocated, using FSE_createDTable().
return : 0, or an errorCode, which can be tested using FSE_isError() */
size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog);
/*! FSE_decompress_usingDTable():
Decompress compressed source `cSrc` of size `cSrcSize` using `dt`
into `dst` which must be already allocated.
@return : size of regenerated data (necessarily <= `dstCapacity`),
or an errorCode, which can be tested using FSE_isError() */
size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt);
/*!
Tutorial :
----------
(Note : these functions only decompress FSE-compressed blocks.
If block is uncompressed, use memcpy() instead
If block is a single repeated byte, use memset() instead )
The first step is to obtain the normalized frequencies of symbols.
This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount().
'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short.
In practice, that means it's necessary to know 'maxSymbolValue' beforehand,
or size the table to handle worst case situations (typically 256).
FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'.
The result of FSE_readNCount() is the number of bytes read from 'rBuffer'.
Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that.
If there is an error, the function will return an error code, which can be tested using FSE_isError().
The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'.
This is performed by the function FSE_buildDTable().
The space required by 'FSE_DTable' must be already allocated using FSE_createDTable().
If there is an error, the function will return an error code, which can be tested using FSE_isError().
`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable().
`cSrcSize` must be strictly correct, otherwise decompression will fail.
FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`).
If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small)
*/
#ifdef FSE_STATIC_LINKING_ONLY
/* *** Dependency *** */
#include "bitstream.h"
/* *****************************************
* Static allocation
*******************************************/
/* FSE buffer bounds */
#define FSE_NCOUNTBOUND 512
#define FSE_BLOCKBOUND(size) (size + (size>>7))
#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* It is possible to statically allocate FSE CTable/DTable as a table of unsigned using below macros */
#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2))
#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<maxTableLog))
/* *****************************************
* FSE advanced API
*******************************************/
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize);
/**< same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr */
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus);
/**< same as FSE_optimalTableLog(), which used `minus==2` */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits);
/**< build a fake FSE_CTable, designed to not compress an input, where each symbol uses nbBits */
size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue);
/**< build a fake FSE_CTable, designed to compress always the same symbolValue */
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits);
/**< build a fake FSE_DTable, designed to read an uncompressed bitstream where each symbol uses nbBits */
size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue);
/**< build a fake FSE_DTable, designed to always generate the same symbolValue */
/* *****************************************
* FSE symbol compression API
*******************************************/
/*!
This API consists of small unitary functions, which highly benefit from being inlined.
You will want to enable link-time-optimization to ensure these functions are properly inlined in your binary.
Visual seems to do it automatically.
For gcc or clang, you'll need to add -flto flag at compilation and linking stages.
If none of these solutions is applicable, include "fse.c" directly.
*/
typedef struct
{
ptrdiff_t value;
const void* stateTable;
const void* symbolTT;
unsigned stateLog;
} FSE_CState_t;
static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct);
static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol);
static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr);
/**<
These functions are inner components of FSE_compress_usingCTable().
They allow the creation of custom streams, mixing multiple tables and bit sources.
A key property to keep in mind is that encoding and decoding are done **in reverse direction**.
So the first symbol you will encode is the last you will decode, like a LIFO stack.
You will need a few variables to track your CStream. They are :
FSE_CTable ct; // Provided by FSE_buildCTable()
BIT_CStream_t bitStream; // bitStream tracking structure
FSE_CState_t state; // State tracking structure (can have several)
The first thing to do is to init bitStream and state.
size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize);
FSE_initCState(&state, ct);
Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError();
You can then encode your input data, byte after byte.
FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time.
Remember decoding will be done in reverse direction.
FSE_encodeByte(&bitStream, &state, symbol);
At any time, you can also add any bit sequence.
Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders
BIT_addBits(&bitStream, bitField, nbBits);
The above methods don't commit data to memory, they just store it into local register, for speed.
Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t).
Writing data to memory is a manual operation, performed by the flushBits function.
BIT_flushBits(&bitStream);
Your last FSE encoding operation shall be to flush your last state value(s).
FSE_flushState(&bitStream, &state);
Finally, you must close the bitStream.
The function returns the size of CStream in bytes.
If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible)
If there is an error, it returns an errorCode (which can be tested using FSE_isError()).
size_t size = BIT_closeCStream(&bitStream);
*/
/* *****************************************
* FSE symbol decompression API
*******************************************/
typedef struct
{
size_t state;
const void* table; /* precise table may vary, depending on U16 */
} FSE_DState_t;
static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt);
static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr);
/**<
Let's now decompose FSE_decompress_usingDTable() into its unitary components.
You will decode FSE-encoded symbols from the bitStream,
and also any other bitFields you put in, **in reverse order**.
You will need a few variables to track your bitStream. They are :
BIT_DStream_t DStream; // Stream context
FSE_DState_t DState; // State context. Multiple ones are possible
FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable()
The first thing to do is to init the bitStream.
errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize);
You should then retrieve your initial state(s)
(in reverse flushing order if you have several ones) :
errorCode = FSE_initDState(&DState, &DStream, DTablePtr);
You can then decode your data, symbol after symbol.
For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'.
Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out).
unsigned char symbol = FSE_decodeSymbol(&DState, &DStream);
You can retrieve any bitfield you eventually stored into the bitStream (in reverse order)
Note : maximum allowed nbBits is 25, for 32-bits compatibility
size_t bitField = BIT_readBits(&DStream, nbBits);
All above operations only read from local register (which size depends on size_t).
Refueling the register from memory is manually performed by the reload method.
endSignal = FSE_reloadDStream(&DStream);
BIT_reloadDStream() result tells if there is still some more data to read from DStream.
BIT_DStream_unfinished : there is still some data left into the DStream.
BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled.
BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed.
BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted.
When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop,
to properly detect the exact end of stream.
After each decoded symbol, check if DStream is fully consumed using this simple test :
BIT_reloadDStream(&DStream) >= BIT_DStream_completed
When it's done, verify decompression is fully completed, by checking both DStream and the relevant states.
Checking if DStream has reached its end is performed by :
BIT_endOfDStream(&DStream);
Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible.
FSE_endOfDState(&DState);
*/
/* *****************************************
* FSE unsafe API
*******************************************/
static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD);
/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */
/* *****************************************
* Implementation of inlined functions
*******************************************/
typedef struct {
int deltaFindState;
U32 deltaNbBits;
} FSE_symbolCompressionTransform; /* total 8 bytes */
MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
{
const void* ptr = ct;
const U16* u16ptr = (const U16*) ptr;
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->stateLog = tableLog;
}
/*! FSE_initCState2() :
* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read)
* uses the smallest state value possible, saving the cost of this symbol */
MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol)
{
FSE_initCState(statePtr, ct);
{ const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16);
statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits;
statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
}
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
{
const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);
U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16);
BIT_addBits(bitC, statePtr->value, nbBitsOut);
statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState];
}
MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr)
{
BIT_addBits(bitC, statePtr->value, statePtr->stateLog);
BIT_flushBits(bitC);
}
/* ====== Decompression ====== */
typedef struct {
U16 tableLog;
U16 fastMode;
} FSE_DTableHeader; /* sizeof U32 */
typedef struct
{
unsigned short newState;
unsigned char symbol;
unsigned char nbBits;
} FSE_decode_t; /* size == U32 */
MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt)
{
const void* ptr = dt;
const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr;
DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog);
BIT_reloadDStream(bitD);
DStatePtr->table = dt + 1;
}
MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
return DInfo.symbol;
}
MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
}
MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBits(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
/*! FSE_decodeSymbolFast() :
unsafe, only works if no symbol has a probability > 50% */
MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD)
{
FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state];
U32 const nbBits = DInfo.nbBits;
BYTE const symbol = DInfo.symbol;
size_t const lowBits = BIT_readBitsFast(bitD, nbBits);
DStatePtr->state = DInfo.newState + lowBits;
return symbol;
}
MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
{
return DStatePtr->state == 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/* **************************************************************
* Tuning parameters
****************************************************************/
/*!MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#define FSE_MAX_MEMORY_USAGE 14
#define FSE_DEFAULT_MEMORY_USAGE 13
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#define FSE_MAX_SYMBOL_VALUE 255
/* **************************************************************
* template functions type & suffix
****************************************************************/
#define FSE_FUNCTION_TYPE BYTE
#define FSE_FUNCTION_EXTENSION
#define FSE_DECODE_TYPE FSE_decode_t
#endif /* !FSE_COMMONDEFS_ONLY */
/* ***************************************************************
* Constants
*****************************************************************/
#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2)
#define FSE_MAX_TABLESIZE (1U<<FSE_MAX_TABLELOG)
#define FSE_MAXTABLESIZE_MASK (FSE_MAX_TABLESIZE-1)
#define FSE_DEFAULT_TABLELOG (FSE_DEFAULT_MEMORY_USAGE-2)
#define FSE_MIN_TABLELOG 5
#define FSE_TABLELOG_ABSOLUTE_MAX 15
#if FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX
# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported"
#endif
#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3)
#endif /* FSE_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* FSE_H */
C/lizard/fse_compress.c
+810
View File
@@ -0,0 +1,810 @@
/* ******************************************************************
FSE : Finite State Entropy encoder
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* **************************************************************
* Error Management
****************************************************************/
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* **************************************************************
* Complex types
****************************************************************/
typedef U32 CTable_max_t[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)];
/* **************************************************************
* Templates
****************************************************************/
/*
designed to be included
for type-specific functions (template emulation in C)
Objective is to write these functions only once, for improved maintenance
*/
/* safety checks */
#ifndef FSE_FUNCTION_EXTENSION
# error "FSE_FUNCTION_EXTENSION must be defined"
#endif
#ifndef FSE_FUNCTION_TYPE
# error "FSE_FUNCTION_TYPE must be defined"
#endif
/* Function names */
#define FSE_CAT(X,Y) X##Y
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
/* Function templates */
size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
U32 const tableSize = 1 << tableLog;
U32 const tableMask = tableSize - 1;
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableLog ? tableSize>>1 : 1) ;
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
U32 const step = FSE_TABLESTEP(tableSize);
U32 cumul[FSE_MAX_SYMBOL_VALUE+2];
FSE_FUNCTION_TYPE tableSymbol[FSE_MAX_TABLESIZE]; /* memset() is not necessary, even if static analyzer complain about it */
U32 highThreshold = tableSize-1;
/* CTable header */
tableU16[-2] = (U16) tableLog;
tableU16[-1] = (U16) maxSymbolValue;
/* For explanations on how to distribute symbol values over the table :
* http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */
/* symbol start positions */
{ U32 u;
cumul[0] = 0;
for (u=1; u<=maxSymbolValue+1; u++) {
if (normalizedCounter[u-1]==-1) { /* Low proba symbol */
cumul[u] = cumul[u-1] + 1;
tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1);
} else {
cumul[u] = cumul[u-1] + normalizedCounter[u-1];
} }
cumul[maxSymbolValue+1] = tableSize+1;
}
/* Spread symbols */
{ U32 position = 0;
U32 symbol;
for (symbol=0; symbol<=maxSymbolValue; symbol++) {
int nbOccurences;
for (nbOccurences=0; nbOccurences<normalizedCounter[symbol]; nbOccurences++) {
tableSymbol[position] = (FSE_FUNCTION_TYPE)symbol;
position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* Low proba area */
} }
if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */
}
/* Build table */
{ U32 u; for (u=0; u<tableSize; u++) {
FSE_FUNCTION_TYPE s = tableSymbol[u]; /* note : static analyzer may not understand tableSymbol is properly initialized */
tableU16[cumul[s]++] = (U16) (tableSize+u); /* TableU16 : sorted by symbol order; gives next state value */
} }
/* Build Symbol Transformation Table */
{ unsigned total = 0;
unsigned s;
for (s=0; s<=maxSymbolValue; s++) {
switch (normalizedCounter[s])
{
case 0: break;
case -1:
case 1:
symbolTT[s].deltaNbBits = (tableLog << 16) - (1<<tableLog);
symbolTT[s].deltaFindState = total - 1;
total ++;
break;
default :
{
U32 const maxBitsOut = tableLog - BIT_highbit32 (normalizedCounter[s]-1);
U32 const minStatePlus = normalizedCounter[s] << maxBitsOut;
symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus;
symbolTT[s].deltaFindState = total - normalizedCounter[s];
total += normalizedCounter[s];
} } } }
return 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/*-**************************************************************
* FSE NCount encoding-decoding
****************************************************************/
size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog)
{
size_t maxHeaderSize = (((maxSymbolValue+1) * tableLog) >> 3) + 3;
return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */
}
static short FSE_abs(short a) { return (short)(a<0 ? -a : a); }
static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize,
const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog,
unsigned writeIsSafe)
{
BYTE* const ostart = (BYTE*) header;
BYTE* out = ostart;
BYTE* const oend = ostart + headerBufferSize;
int nbBits;
const int tableSize = 1 << tableLog;
int remaining;
int threshold;
U32 bitStream;
int bitCount;
unsigned charnum = 0;
int previous0 = 0;
bitStream = 0;
bitCount = 0;
/* Table Size */
bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount;
bitCount += 4;
/* Init */
remaining = tableSize+1; /* +1 for extra accuracy */
threshold = tableSize;
nbBits = tableLog+1;
while (remaining>1) { /* stops at 1 */
if (previous0) {
unsigned start = charnum;
while (!normalizedCounter[charnum]) charnum++;
while (charnum >= start+24) {
start+=24;
bitStream += 0xFFFFU << bitCount;
if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE) bitStream;
out[1] = (BYTE)(bitStream>>8);
out+=2;
bitStream>>=16;
}
while (charnum >= start+3) {
start+=3;
bitStream += 3 << bitCount;
bitCount += 2;
}
bitStream += (charnum-start) << bitCount;
bitCount += 2;
if (bitCount>16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
bitStream >>= 16;
bitCount -= 16;
} }
{ short count = normalizedCounter[charnum++];
const short max = (short)((2*threshold-1)-remaining);
remaining -= FSE_abs(count);
if (remaining<1) return ERROR(GENERIC);
count++; /* +1 for extra accuracy */
if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */
bitStream += count << bitCount;
bitCount += nbBits;
bitCount -= (count<max);
previous0 = (count==1);
while (remaining<threshold) nbBits--, threshold>>=1;
}
if (bitCount>16) {
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out += 2;
bitStream >>= 16;
bitCount -= 16;
} }
/* flush remaining bitStream */
if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */
out[0] = (BYTE)bitStream;
out[1] = (BYTE)(bitStream>>8);
out+= (bitCount+7) /8;
if (charnum > maxSymbolValue + 1) return ERROR(GENERIC);
return (out-ostart);
}
size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */
if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog))
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0);
return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1);
}
/*-**************************************************************
* Counting histogram
****************************************************************/
/*! FSE_count_simple
This function just counts byte values within `src`,
and store the histogram into table `count`.
This function is unsafe : it doesn't check that all values within `src` can fit into `count`.
For this reason, prefer using a table `count` with 256 elements.
@return : count of most numerous element
*/
static size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{
const BYTE* ip = (const BYTE*)src;
const BYTE* const end = ip + srcSize;
unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0;
memset(count, 0, (maxSymbolValue+1)*sizeof(*count));
if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; }
while (ip<end) count[*ip++]++;
while (!count[maxSymbolValue]) maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue;
{ U32 s; for (s=0; s<=maxSymbolValue; s++) if (count[s] > max) max = count[s]; }
return (size_t)max;
}
static size_t FSE_count_parallel(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
unsigned checkMax)
{
const BYTE* ip = (const BYTE*)source;
const BYTE* const iend = ip+sourceSize;
unsigned maxSymbolValue = *maxSymbolValuePtr;
unsigned max=0;
U32 Counting1[256] = { 0 };
U32 Counting2[256] = { 0 };
U32 Counting3[256] = { 0 };
U32 Counting4[256] = { 0 };
/* safety checks */
if (!sourceSize) {
memset(count, 0, maxSymbolValue + 1);
*maxSymbolValuePtr = 0;
return 0;
}
if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */
/* by stripes of 16 bytes */
{ U32 cached = MEM_read32(ip); ip += 4;
while (ip < iend-15) {
U32 c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
c = cached; cached = MEM_read32(ip); ip += 4;
Counting1[(BYTE) c ]++;
Counting2[(BYTE)(c>>8) ]++;
Counting3[(BYTE)(c>>16)]++;
Counting4[ c>>24 ]++;
}
ip-=4;
}
/* finish last symbols */
while (ip<iend) Counting1[*ip++]++;
if (checkMax) { /* verify stats will fit into destination table */
U32 s; for (s=255; s>maxSymbolValue; s--) {
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
} }
{ U32 s; for (s=0; s<=maxSymbolValue; s++) {
count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s];
if (count[s] > max) max = count[s];
}}
while (!count[maxSymbolValue]) maxSymbolValue--;
*maxSymbolValuePtr = maxSymbolValue;
return (size_t)max;
}
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize)
{
if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize);
return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 0);
}
size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize)
{
if (*maxSymbolValuePtr <255)
return FSE_count_parallel(count, maxSymbolValuePtr, source, sourceSize, 1);
*maxSymbolValuePtr = 255;
return FSE_countFast(count, maxSymbolValuePtr, source, sourceSize);
}
/*-**************************************************************
* FSE Compression Code
****************************************************************/
/*! FSE_sizeof_CTable() :
FSE_CTable is a variable size structure which contains :
`U16 tableLog;`
`U16 maxSymbolValue;`
`U16 nextStateNumber[1 << tableLog];` // This size is variable
`FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable
Allocation is manual (C standard does not support variable-size structures).
*/
size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog)
{
size_t size;
FSE_STATIC_ASSERT((size_t)FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)*4 >= sizeof(CTable_max_t)); /* A compilation error here means FSE_CTABLE_SIZE_U32 is not large enough */
if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC);
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
return size;
}
FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog)
{
size_t size;
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32);
return (FSE_CTable*)malloc(size);
}
void FSE_freeCTable (FSE_CTable* ct) { free(ct); }
/* provides the minimum logSize to safely represent a distribution */
static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue)
{
U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1;
U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2;
U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols;
return minBits;
}
unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus)
{
U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus;
U32 tableLog = maxTableLog;
U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue);
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */
if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */
if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG;
if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG;
return tableLog;
}
unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2);
}
/* Secondary normalization method.
To be used when primary method fails. */
static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue)
{
U32 s;
U32 distributed = 0;
U32 ToDistribute;
/* Init */
U32 lowThreshold = (U32)(total >> tableLog);
U32 lowOne = (U32)((total * 3) >> (tableLog + 1));
for (s=0; s<=maxSymbolValue; s++) {
if (count[s] == 0) {
norm[s]=0;
continue;
}
if (count[s] <= lowThreshold) {
norm[s] = -1;
distributed++;
total -= count[s];
continue;
}
if (count[s] <= lowOne) {
norm[s] = 1;
distributed++;
total -= count[s];
continue;
}
norm[s]=-2;
}
ToDistribute = (1 << tableLog) - distributed;
if ((total / ToDistribute) > lowOne) {
/* risk of rounding to zero */
lowOne = (U32)((total * 3) / (ToDistribute * 2));
for (s=0; s<=maxSymbolValue; s++) {
if ((norm[s] == -2) && (count[s] <= lowOne)) {
norm[s] = 1;
distributed++;
total -= count[s];
continue;
} }
ToDistribute = (1 << tableLog) - distributed;
}
if (distributed == maxSymbolValue+1) {
/* all values are pretty poor;
probably incompressible data (should have already been detected);
find max, then give all remaining points to max */
U32 maxV = 0, maxC = 0;
for (s=0; s<=maxSymbolValue; s++)
if (count[s] > maxC) maxV=s, maxC=count[s];
norm[maxV] += (short)ToDistribute;
return 0;
}
{
U64 const vStepLog = 62 - tableLog;
U64 const mid = (1ULL << (vStepLog-1)) - 1;
U64 const rStep = ((((U64)1<<vStepLog) * ToDistribute) + mid) / total; /* scale on remaining */
U64 tmpTotal = mid;
for (s=0; s<=maxSymbolValue; s++) {
if (norm[s]==-2) {
U64 end = tmpTotal + (count[s] * rStep);
U32 sStart = (U32)(tmpTotal >> vStepLog);
U32 sEnd = (U32)(end >> vStepLog);
U32 weight = sEnd - sStart;
if (weight < 1)
return ERROR(GENERIC);
norm[s] = (short)weight;
tmpTotal = end;
} } }
return 0;
}
size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog,
const unsigned* count, size_t total,
unsigned maxSymbolValue)
{
/* Sanity checks */
if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG;
if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */
if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */
{ U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 };
U64 const scale = 62 - tableLog;
U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */
U64 const vStep = 1ULL<<(scale-20);
int stillToDistribute = 1<<tableLog;
unsigned s;
unsigned largest=0;
short largestP=0;
U32 lowThreshold = (U32)(total >> tableLog);
for (s=0; s<=maxSymbolValue; s++) {
if (count[s] == total) return 0; /* rle special case */
if (count[s] == 0) { normalizedCounter[s]=0; continue; }
if (count[s] <= lowThreshold) {
normalizedCounter[s] = -1;
stillToDistribute--;
} else {
short proba = (short)((count[s]*step) >> scale);
if (proba<8) {
U64 restToBeat = vStep * rtbTable[proba];
proba += (count[s]*step) - ((U64)proba<<scale) > restToBeat;
}
if (proba > largestP) largestP=proba, largest=s;
normalizedCounter[s] = proba;
stillToDistribute -= proba;
} }
if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) {
/* corner case, need another normalization method */
size_t errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
}
else normalizedCounter[largest] += (short)stillToDistribute;
}
#if 0
{ /* Print Table (debug) */
U32 s;
U32 nTotal = 0;
for (s=0; s<=maxSymbolValue; s++)
printf("%3i: %4i \n", s, normalizedCounter[s]);
for (s=0; s<=maxSymbolValue; s++)
nTotal += abs(normalizedCounter[s]);
if (nTotal != (1U<<tableLog))
printf("Warning !!! Total == %u != %u !!!", nTotal, 1U<<tableLog);
getchar();
}
#endif
return tableLog;
}
/* fake FSE_CTable, for raw (uncompressed) input */
size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits)
{
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSymbolValue = tableMask;
void* const ptr = ct;
U16* const tableU16 = ( (U16*) ptr) + 2;
void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */
FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT);
unsigned s;
/* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */
/* header */
tableU16[-2] = (U16) nbBits;
tableU16[-1] = (U16) maxSymbolValue;
/* Build table */
for (s=0; s<tableSize; s++)
tableU16[s] = (U16)(tableSize + s);
/* Build Symbol Transformation Table */
{ const U32 deltaNbBits = (nbBits << 16) - (1 << nbBits);
for (s=0; s<=maxSymbolValue; s++) {
symbolTT[s].deltaNbBits = deltaNbBits;
symbolTT[s].deltaFindState = s-1;
} }
return 0;
}
/* fake FSE_CTable, for rle (100% always same symbol) input */
size_t FSE_buildCTable_rle (FSE_CTable* ct, BYTE symbolValue)
{
void* ptr = ct;
U16* tableU16 = ( (U16*) ptr) + 2;
void* FSCTptr = (U32*)ptr + 2;
FSE_symbolCompressionTransform* symbolTT = (FSE_symbolCompressionTransform*) FSCTptr;
/* header */
tableU16[-2] = (U16) 0;
tableU16[-1] = (U16) symbolValue;
/* Build table */
tableU16[0] = 0;
tableU16[1] = 0; /* just in case */
/* Build Symbol Transformation Table */
symbolTT[symbolValue].deltaNbBits = 0;
symbolTT[symbolValue].deltaFindState = 0;
return 0;
}
static size_t FSE_compress_usingCTable_generic (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct, const unsigned fast)
{
const BYTE* const istart = (const BYTE*) src;
const BYTE* const iend = istart + srcSize;
const BYTE* ip=iend;
BIT_CStream_t bitC;
FSE_CState_t CState1, CState2;
/* init */
if (srcSize <= 2) return 0;
{ size_t const errorCode = BIT_initCStream(&bitC, dst, dstSize);
if (FSE_isError(errorCode)) return 0; }
#define FSE_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
if (srcSize & 1) {
FSE_initCState2(&CState1, ct, *--ip);
FSE_initCState2(&CState2, ct, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_FLUSHBITS(&bitC);
} else {
FSE_initCState2(&CState2, ct, *--ip);
FSE_initCState2(&CState1, ct, *--ip);
}
/* join to mod 4 */
srcSize -= 2;
if ((sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
FSE_FLUSHBITS(&bitC);
}
/* 2 or 4 encoding per loop */
for ( ; ip>istart ; ) {
FSE_encodeSymbol(&bitC, &CState2, *--ip);
if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */
FSE_FLUSHBITS(&bitC);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */
FSE_encodeSymbol(&bitC, &CState2, *--ip);
FSE_encodeSymbol(&bitC, &CState1, *--ip);
}
FSE_FLUSHBITS(&bitC);
}
FSE_flushCState(&bitC, &CState2);
FSE_flushCState(&bitC, &CState1);
return BIT_closeCStream(&bitC);
}
size_t FSE_compress_usingCTable (void* dst, size_t dstSize,
const void* src, size_t srcSize,
const FSE_CTable* ct)
{
const unsigned fast = (dstSize >= FSE_BLOCKBOUND(srcSize));
if (fast)
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1);
else
return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0);
}
size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); }
size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog)
{
const BYTE* const istart = (const BYTE*) src;
const BYTE* ip = istart;
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 count[FSE_MAX_SYMBOL_VALUE+1];
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
CTable_max_t ct;
size_t errorCode;
/* init conditions */
if (srcSize <= 1) return 0; /* Uncompressible */
if (!maxSymbolValue) maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */
errorCode = FSE_count (count, &maxSymbolValue, ip, srcSize);
if (FSE_isError(errorCode)) return errorCode;
if (errorCode == srcSize) return 1;
if (errorCode == 1) return 0; /* each symbol only present once */
if (errorCode < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */
tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue);
errorCode = FSE_normalizeCount (norm, tableLog, count, srcSize, maxSymbolValue);
if (FSE_isError(errorCode)) return errorCode;
/* Write table description header */
errorCode = FSE_writeNCount (op, oend-op, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
op += errorCode;
/* Compress */
errorCode = FSE_buildCTable (ct, norm, maxSymbolValue, tableLog);
if (FSE_isError(errorCode)) return errorCode;
errorCode = FSE_compress_usingCTable(op, oend - op, ip, srcSize, ct);
if (errorCode == 0) return 0; /* not enough space for compressed data */
op += errorCode;
/* check compressibility */
if ( (size_t)(op-ostart) >= srcSize-1 )
return 0;
return op-ostart;
}
size_t FSE_compress (void* dst, size_t dstSize, const void* src, size_t srcSize)
{
return FSE_compress2(dst, dstSize, src, (U32)srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG);
}
#endif /* FSE_COMMONDEFS_ONLY */
C/lizard/fse_decompress.c
+329
View File
@@ -0,0 +1,329 @@
/* ******************************************************************
FSE : Finite State Entropy decoder
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/* **************************************************************
* Includes
****************************************************************/
#include <stdlib.h> /* malloc, free, qsort */
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
/* **************************************************************
* Error Management
****************************************************************/
#define FSE_isError ERR_isError
#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* check and forward error code */
#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; }
/* **************************************************************
* Complex types
****************************************************************/
typedef U32 DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)];
/* **************************************************************
* Templates
****************************************************************/
/*
designed to be included
for type-specific functions (template emulation in C)
Objective is to write these functions only once, for improved maintenance
*/
/* safety checks */
#ifndef FSE_FUNCTION_EXTENSION
# error "FSE_FUNCTION_EXTENSION must be defined"
#endif
#ifndef FSE_FUNCTION_TYPE
# error "FSE_FUNCTION_TYPE must be defined"
#endif
/* Function names */
#define FSE_CAT(X,Y) X##Y
#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y)
#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y)
/* Function templates */
FSE_DTable* FSE_createDTable (unsigned tableLog)
{
if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX;
return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) );
}
void FSE_freeDTable (FSE_DTable* dt)
{
free(dt);
}
size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog)
{
void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */
FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr);
U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1];
U32 const maxSV1 = maxSymbolValue + 1;
U32 const tableSize = 1 << tableLog;
U32 highThreshold = tableSize-1;
/* Sanity Checks */
if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge);
if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge);
/* Init, lay down lowprob symbols */
{ FSE_DTableHeader DTableH;
DTableH.tableLog = (U16)tableLog;
DTableH.fastMode = 1;
{ S16 const largeLimit= (S16)(1 << (tableLog-1));
U32 s;
for (s=0; s<maxSV1; s++) {
if (normalizedCounter[s]==-1) {
tableDecode[highThreshold--].symbol = (FSE_FUNCTION_TYPE)s;
symbolNext[s] = 1;
} else {
if (normalizedCounter[s] >= largeLimit) DTableH.fastMode=0;
symbolNext[s] = normalizedCounter[s];
} } }
memcpy(dt, &DTableH, sizeof(DTableH));
}
/* Spread symbols */
{ U32 const tableMask = tableSize-1;
U32 const step = FSE_TABLESTEP(tableSize);
U32 s, position = 0;
for (s=0; s<maxSV1; s++) {
int i;
for (i=0; i<normalizedCounter[s]; i++) {
tableDecode[position].symbol = (FSE_FUNCTION_TYPE)s;
position = (position + step) & tableMask;
while (position > highThreshold) position = (position + step) & tableMask; /* lowprob area */
} }
if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */
}
/* Build Decoding table */
{ U32 u;
for (u=0; u<tableSize; u++) {
FSE_FUNCTION_TYPE const symbol = (FSE_FUNCTION_TYPE)(tableDecode[u].symbol);
U16 nextState = symbolNext[symbol]++;
tableDecode[u].nbBits = (BYTE) (tableLog - BIT_highbit32 ((U32)nextState) );
tableDecode[u].newState = (U16) ( (nextState << tableDecode[u].nbBits) - tableSize);
} }
return 0;
}
#ifndef FSE_COMMONDEFS_ONLY
/*-*******************************************************
* Decompression (Byte symbols)
*********************************************************/
size_t FSE_buildDTable_rle (FSE_DTable* dt, BYTE symbolValue)
{
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const cell = (FSE_decode_t*)dPtr;
DTableH->tableLog = 0;
DTableH->fastMode = 0;
cell->newState = 0;
cell->symbol = symbolValue;
cell->nbBits = 0;
return 0;
}
size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits)
{
void* ptr = dt;
FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr;
void* dPtr = dt + 1;
FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr;
const unsigned tableSize = 1 << nbBits;
const unsigned tableMask = tableSize - 1;
const unsigned maxSV1 = tableMask+1;
unsigned s;
/* Sanity checks */
if (nbBits < 1) return ERROR(GENERIC); /* min size */
/* Build Decoding Table */
DTableH->tableLog = (U16)nbBits;
DTableH->fastMode = 1;
for (s=0; s<maxSV1; s++) {
dinfo[s].newState = 0;
dinfo[s].symbol = (BYTE)s;
dinfo[s].nbBits = (BYTE)nbBits;
}
return 0;
}
FORCE_INLINE size_t FSE_decompress_usingDTable_generic(
void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt, const unsigned fast)
{
BYTE* const ostart = (BYTE*) dst;
BYTE* op = ostart;
BYTE* const omax = op + maxDstSize;
BYTE* const olimit = omax-3;
BIT_DStream_t bitD;
FSE_DState_t state1;
FSE_DState_t state2;
/* Init */
CHECK_F(BIT_initDStream(&bitD, cSrc, cSrcSize));
FSE_initDState(&state1, &bitD, dt);
FSE_initDState(&state2, &bitD, dt);
#define FSE_GETSYMBOL(statePtr) fast ? FSE_decodeSymbolFast(statePtr, &bitD) : FSE_decodeSymbol(statePtr, &bitD)
/* 4 symbols per loop */
for ( ; (BIT_reloadDStream(&bitD)==BIT_DStream_unfinished) & (op<olimit) ; op+=4) {
op[0] = FSE_GETSYMBOL(&state1);
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
BIT_reloadDStream(&bitD);
op[1] = FSE_GETSYMBOL(&state2);
if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
{ if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } }
op[2] = FSE_GETSYMBOL(&state1);
if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */
BIT_reloadDStream(&bitD);
op[3] = FSE_GETSYMBOL(&state2);
}
/* tail */
/* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */
while (1) {
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state1);
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state2);
break;
}
if (op>(omax-2)) return ERROR(dstSize_tooSmall);
*op++ = FSE_GETSYMBOL(&state2);
if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) {
*op++ = FSE_GETSYMBOL(&state1);
break;
} }
return op-ostart;
}
size_t FSE_decompress_usingDTable(void* dst, size_t originalSize,
const void* cSrc, size_t cSrcSize,
const FSE_DTable* dt)
{
const void* ptr = dt;
const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr;
const U32 fastMode = DTableH->fastMode;
/* select fast mode (static) */
if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1);
return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0);
}
size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* const istart = (const BYTE*)cSrc;
const BYTE* ip = istart;
short counting[FSE_MAX_SYMBOL_VALUE+1];
DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */
unsigned tableLog;
unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE;
if (cSrcSize<2) return ERROR(srcSize_wrong); /* too small input size */
/* normal FSE decoding mode */
{ size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize);
if (FSE_isError(NCountLength)) return NCountLength;
if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size */
ip += NCountLength;
cSrcSize -= NCountLength;
}
CHECK_F( FSE_buildDTable (dt, counting, maxSymbolValue, tableLog) );
return FSE_decompress_usingDTable (dst, maxDstSize, ip, cSrcSize, dt); /* always return, even if it is an error code */
}
#endif /* FSE_COMMONDEFS_ONLY */
C/lizard/huf.h
+228
View File
@@ -0,0 +1,228 @@
/* ******************************************************************
Huffman coder, part of New Generation Entropy library
header file
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef HUF_H_298734234
#define HUF_H_298734234
#if defined (__cplusplus)
extern "C" {
#endif
/* *** Dependencies *** */
#include <stddef.h> /* size_t */
/* *** simple functions *** */
/**
HUF_compress() :
Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'.
'dst' buffer must be already allocated.
Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize).
`srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB.
@return : size of compressed data (<= `dstCapacity`).
Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!!
if return == 1, srcData is a single repeated byte symbol (RLE compression).
if HUF_isError(return), compression failed (more details using HUF_getErrorName())
*/
size_t HUF_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/**
HUF_decompress() :
Decompress HUF data from buffer 'cSrc', of size 'cSrcSize',
into already allocated buffer 'dst', of minimum size 'dstSize'.
`dstSize` : **must** be the ***exact*** size of original (uncompressed) data.
Note : in contrast with FSE, HUF_decompress can regenerate
RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data,
because it knows size to regenerate.
@return : size of regenerated data (== dstSize),
or an error code, which can be tested using HUF_isError()
*/
size_t HUF_decompress(void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize);
/* ****************************************
* Tool functions
******************************************/
#define HUF_BLOCKSIZE_MAX (128 * 1024)
size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */
/* Error Management */
unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */
const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */
/* *** Advanced function *** */
/** HUF_compress2() :
* Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` */
size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
#ifdef HUF_STATIC_LINKING_ONLY
/* *** Dependencies *** */
#include "mem.h" /* U32 */
/* *** Constants *** */
#define HUF_TABLELOG_ABSOLUTEMAX 16 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */
#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */
#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */
#define HUF_SYMBOLVALUE_MAX 255
#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX)
# error "HUF_TABLELOG_MAX is too large !"
#endif
/* ****************************************
* Static allocation
******************************************/
/* HUF buffer bounds */
#define HUF_CTABLEBOUND 129
#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */
#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */
/* static allocation of HUF's Compression Table */
#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \
U32 name##hb[maxSymbolValue+1]; \
void* name##hv = &(name##hb); \
HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */
/* static allocation of HUF's DTable */
typedef U32 HUF_DTable;
#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog)))
#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1)*0x1000001) }
#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \
HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog)*0x1000001) }
/* ****************************************
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
/* ****************************************
* HUF detailed API
******************************************/
/*!
HUF_compress() does the following:
1. count symbol occurrence from source[] into table count[] using FSE_count()
2. (optional) refine tableLog using HUF_optimalTableLog()
3. build Huffman table from count using HUF_buildCTable()
4. save Huffman table to memory buffer using HUF_writeCTable()
5. encode the data stream using HUF_compress4X_usingCTable()
The following API allows targeting specific sub-functions for advanced tasks.
For example, it's possible to compress several blocks using the same 'CTable',
or to save and regenerate 'CTable' using external methods.
*/
/* FSE_count() : find it within "fse.h" */
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue);
typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */
size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits);
size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog);
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
/*! HUF_readStats() :
Read compact Huffman tree, saved by HUF_writeCTable().
`huffWeight` is destination buffer.
@return : size read from `src` , or an error Code .
Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */
size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats,
U32* nbSymbolsPtr, U32* tableLogPtr,
const void* src, size_t srcSize);
/** HUF_readCTable() :
* Loading a CTable saved with HUF_writeCTable() */
size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize);
/*
HUF_decompress() does the following:
1. select the decompression algorithm (X2, X4) based on pre-computed heuristics
2. build Huffman table from save, using HUF_readDTableXn()
3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable
*/
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
/* single stream variants */
size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog);
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable);
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif /* HUF_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* HUF_H_298734234 */
C/lizard/huf_compress.c
+532
View File
@@ -0,0 +1,532 @@
/* ******************************************************************
Huffman encoder, part of New Generation Entropy library
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* **************************************************************
* Includes
****************************************************************/
#include <string.h> /* memcpy, memset */
#include <stdio.h> /* printf (debug) */
#include "bitstream.h"
#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */
#include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
/* **************************************************************
* Error Management
****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* **************************************************************
* Utils
****************************************************************/
unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue)
{
return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1);
}
/* *******************************************************
* HUF : Huffman block compression
*********************************************************/
struct HUF_CElt_s {
U16 val;
BYTE nbBits;
}; /* typedef'd to HUF_CElt within "huf.h" */
typedef struct nodeElt_s {
U32 count;
U16 parent;
BYTE byte;
BYTE nbBits;
} nodeElt;
/*! HUF_writeCTable() :
`CTable` : huffman tree to save, using huf representation.
@return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
{
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1];
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
BYTE* op = (BYTE*)dst;
U32 n;
/* check conditions */
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
/* convert to weight */
bitsToWeight[0] = 0;
for (n=1; n<huffLog+1; n++)
bitsToWeight[n] = (BYTE)(huffLog + 1 - n);
for (n=0; n<maxSymbolValue; n++)
huffWeight[n] = bitsToWeight[CTable[n].nbBits];
{ size_t const size = FSE_compress(op+1, maxDstSize-1, huffWeight, maxSymbolValue);
if (FSE_isError(size)) return size;
if ((size>1) & (size < maxSymbolValue/2)) { /* FSE compressed */
op[0] = (BYTE)size;
return size+1;
}
}
/* raw values */
if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen */
if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */
op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1));
huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause issue in final combination */
for (n=0; n<maxSymbolValue; n+=2)
op[(n/2)+1] = (BYTE)((huffWeight[n] << 4) + huffWeight[n+1]);
return ((maxSymbolValue+1)/2) + 1;
}
size_t HUF_readCTable (HUF_CElt* CTable, U32 maxSymbolValue, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
size_t readSize;
U32 nbSymbols = 0;
/*memset(huffWeight, 0, sizeof(huffWeight));*/ /* is not necessary, even though some analyzer complain ... */
/* get symbol weights */
readSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX+1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(readSize)) return readSize;
/* check result */
if (tableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall);
/* Prepare base value per rank */
{ U32 n, nextRankStart = 0;
for (n=1; n<=tableLog; n++) {
U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current;
} }
/* fill nbBits */
{ U32 n; for (n=0; n<nbSymbols; n++) {
const U32 w = huffWeight[n];
CTable[n].nbBits = (BYTE)(tableLog + 1 - w);
} }
/* fill val */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
{ U32 n; for (n=0; n<nbSymbols; n++) nbPerRank[CTable[n].nbBits]++; }
/* determine stating value per rank */
{ U16 min = 0;
U32 n; for (n=HUF_TABLELOG_MAX; n>0; n--) {
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
/* assign value within rank, symbol order */
{ U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; }
}
return readSize;
}
static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits)
{
const U32 largestBits = huffNode[lastNonNull].nbBits;
if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */
/* there are several too large elements (at least >= 2) */
{ int totalCost = 0;
const U32 baseCost = 1 << (largestBits - maxNbBits);
U32 n = lastNonNull;
while (huffNode[n].nbBits > maxNbBits) {
totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits));
huffNode[n].nbBits = (BYTE)maxNbBits;
n --;
} /* n stops at huffNode[n].nbBits <= maxNbBits */
while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */
/* renorm totalCost */
totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */
/* repay normalized cost */
{ U32 const noSymbol = 0xF0F0F0F0;
U32 rankLast[HUF_TABLELOG_MAX+2];
int pos;
/* Get pos of last (smallest) symbol per rank */
memset(rankLast, 0xF0, sizeof(rankLast));
{ U32 currentNbBits = maxNbBits;
for (pos=n ; pos >= 0; pos--) {
if (huffNode[pos].nbBits >= currentNbBits) continue;
currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */
rankLast[maxNbBits-currentNbBits] = pos;
} }
while (totalCost > 0) {
U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1;
for ( ; nBitsToDecrease > 1; nBitsToDecrease--) {
U32 highPos = rankLast[nBitsToDecrease];
U32 lowPos = rankLast[nBitsToDecrease-1];
if (highPos == noSymbol) continue;
if (lowPos == noSymbol) break;
{ U32 const highTotal = huffNode[highPos].count;
U32 const lowTotal = 2 * huffNode[lowPos].count;
if (highTotal <= lowTotal) break;
} }
/* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */
while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */
nBitsToDecrease ++;
totalCost -= 1 << (nBitsToDecrease-1);
if (rankLast[nBitsToDecrease-1] == noSymbol)
rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */
huffNode[rankLast[nBitsToDecrease]].nbBits ++;
if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */
rankLast[nBitsToDecrease] = noSymbol;
else {
rankLast[nBitsToDecrease]--;
if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease)
rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */
} } /* while (totalCost > 0) */
while (totalCost < 0) { /* Sometimes, cost correction overshoot */
if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */
while (huffNode[n].nbBits == maxNbBits) n--;
huffNode[n+1].nbBits--;
rankLast[1] = n+1;
totalCost++;
continue;
}
huffNode[ rankLast[1] + 1 ].nbBits--;
rankLast[1]++;
totalCost ++;
} } } /* there are several too large elements (at least >= 2) */
return maxNbBits;
}
typedef struct {
U32 base;
U32 current;
} rankPos;
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
{
rankPos rank[32];
U32 n;
memset(rank, 0, sizeof(rank));
for (n=0; n<=maxSymbolValue; n++) {
U32 r = BIT_highbit32(count[n] + 1);
rank[r].base ++;
}
for (n=30; n>0; n--) rank[n-1].base += rank[n].base;
for (n=0; n<32; n++) rank[n].current = rank[n].base;
for (n=0; n<=maxSymbolValue; n++) {
U32 const c = count[n];
U32 const r = BIT_highbit32(c+1) + 1;
U32 pos = rank[r].current++;
while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--;
huffNode[pos].count = c;
huffNode[pos].byte = (BYTE)n;
}
}
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
{
nodeElt huffNode0[2*HUF_SYMBOLVALUE_MAX+1 +1];
nodeElt* huffNode = huffNode0 + 1;
U32 n, nonNullRank;
int lowS, lowN;
U16 nodeNb = STARTNODE;
U32 nodeRoot;
/* safety checks */
if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT;
if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC);
memset(huffNode0, 0, sizeof(huffNode0));
/* sort, decreasing order */
HUF_sort(huffNode, count, maxSymbolValue);
/* init for parents */
nonNullRank = maxSymbolValue;
while(huffNode[nonNullRank].count == 0) nonNullRank--;
lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb;
huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count;
huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb;
nodeNb++; lowS-=2;
for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30);
huffNode0[0].count = (U32)(1U<<31);
/* create parents */
while (nodeNb <= nodeRoot) {
U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++;
huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count;
huffNode[n1].parent = huffNode[n2].parent = nodeNb;
nodeNb++;
}
/* distribute weights (unlimited tree height) */
huffNode[nodeRoot].nbBits = 0;
for (n=nodeRoot-1; n>=STARTNODE; n--)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
for (n=0; n<=nonNullRank; n++)
huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1;
/* enforce maxTableLog */
maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits);
/* fill result into tree (val, nbBits) */
{ U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0};
U16 valPerRank[HUF_TABLELOG_MAX+1] = {0};
if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */
for (n=0; n<=nonNullRank; n++)
nbPerRank[huffNode[n].nbBits]++;
/* determine stating value per rank */
{ U16 min = 0;
for (n=maxNbBits; n>0; n--) {
valPerRank[n] = min; /* get starting value within each rank */
min += nbPerRank[n];
min >>= 1;
} }
for (n=0; n<=maxSymbolValue; n++)
tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */
for (n=0; n<=maxSymbolValue; n++)
tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */
}
return maxNbBits;
}
static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable)
{
BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits);
}
size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); }
#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s))
#define HUF_FLUSHBITS_1(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream)
#define HUF_FLUSHBITS_2(stream) \
if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream)
size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
const BYTE* ip = (const BYTE*) src;
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
size_t n;
const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize));
BIT_CStream_t bitC;
/* init */
if (dstSize < 8) return 0; /* not enough space to compress */
{ size_t const errorCode = BIT_initCStream(&bitC, op, oend-op);
if (HUF_isError(errorCode)) return 0; }
n = srcSize & ~3; /* join to mod 4 */
switch (srcSize & 3)
{
case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable);
HUF_FLUSHBITS_2(&bitC);
case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable);
HUF_FLUSHBITS_1(&bitC);
case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable);
HUF_FLUSHBITS(&bitC);
case 0 :
default: ;
}
for (; n>0; n-=4) { /* note : n&3==0 at this stage */
HUF_encodeSymbol(&bitC, ip[n- 1], CTable);
HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 2], CTable);
HUF_FLUSHBITS_2(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 3], CTable);
HUF_FLUSHBITS_1(&bitC);
HUF_encodeSymbol(&bitC, ip[n- 4], CTable);
HUF_FLUSHBITS(&bitC);
}
return BIT_closeCStream(&bitC);
}
size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable)
{
size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */
const BYTE* ip = (const BYTE*) src;
const BYTE* const iend = ip + srcSize;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */
if (srcSize < 12) return 0; /* no saving possible : too small input */
op += 6; /* jumpTable */
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
MEM_writeLE16(ostart, (U16)cSize);
op += cSize;
}
ip += segmentSize;
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
MEM_writeLE16(ostart+2, (U16)cSize);
op += cSize;
}
ip += segmentSize;
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
MEM_writeLE16(ostart+4, (U16)cSize);
op += cSize;
}
ip += segmentSize;
{ size_t const cSize = HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0;
op += cSize;
}
return op-ostart;
}
static size_t HUF_compress_internal (
void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
unsigned singleStream)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + dstSize;
BYTE* op = ostart;
U32 count[HUF_SYMBOLVALUE_MAX+1];
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1];
/* checks & inits */
if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */
if (!dstSize) return 0; /* cannot fit within dst budget */
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge);
if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX;
if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT;
/* Scan input and build symbol stats */
{ size_t const largest = FSE_count (count, &maxSymbolValue, (const BYTE*)src, srcSize);
if (HUF_isError(largest)) return largest;
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */
}
/* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ size_t const maxBits = HUF_buildCTable (CTable, count, maxSymbolValue, huffLog);
if (HUF_isError(maxBits)) return maxBits;
huffLog = (U32)maxBits;
}
/* Write table description header */
{ size_t const hSize = HUF_writeCTable (op, dstSize, CTable, maxSymbolValue, huffLog);
if (HUF_isError(hSize)) return hSize;
if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */
op += hSize;
}
/* Compress */
{ size_t const cSize = (singleStream) ?
HUF_compress1X_usingCTable(op, oend - op, src, srcSize, CTable) : /* single segment */
HUF_compress4X_usingCTable(op, oend - op, src, srcSize, CTable);
if (HUF_isError(cSize)) return cSize;
if (cSize==0) return 0; /* uncompressible */
op += cSize;
}
/* check compressibility */
if ((size_t)(op-ostart) >= srcSize-1)
return 0;
return op-ostart;
}
size_t HUF_compress1X (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1);
}
size_t HUF_compress2 (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog)
{
return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0);
}
size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize)
{
return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT);
}
C/lizard/huf_decompress.c
+883
View File
@@ -0,0 +1,883 @@
/* ******************************************************************
Huffman decoder, part of New Generation Entropy library
Copyright (C) 2013-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
/* **************************************************************
* Compiler specifics
****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
/* inline is defined */
#elif defined(_MSC_VER) || defined(__GNUC__)
# define inline __inline
#else
# define inline /* disable inline */
#endif
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#endif
/* **************************************************************
* Dependencies
****************************************************************/
#include <string.h> /* memcpy, memset */
#include "bitstream.h" /* BIT_* */
#include "fse.h" /* header compression */
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
/* **************************************************************
* Error Management
****************************************************************/
#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/*-***************************/
/* generic DTableDesc */
/*-***************************/
typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc;
static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
{
DTableDesc dtd;
memcpy(&dtd, table, sizeof(dtd));
return dtd;
}
/*-***************************/
/* single-symbol decoding */
/*-***************************/
typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1];
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
U32 tableLog = 0;
U32 nbSymbols = 0;
size_t iSize;
void* const dtPtr = DTable + 1;
HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr;
HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable));
/* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* Table header */
{ DTableDesc dtd = HUF_getDTableDesc(DTable);
if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */
dtd.tableType = 0;
dtd.tableLog = (BYTE)tableLog;
memcpy(DTable, &dtd, sizeof(dtd));
}
/* Prepare ranks */
{ U32 n, nextRankStart = 0;
for (n=1; n<tableLog+1; n++) {
U32 current = nextRankStart;
nextRankStart += (rankVal[n] << (n-1));
rankVal[n] = current;
} }
/* fill DTable */
{ U32 n;
for (n=0; n<nbSymbols; n++) {
U32 const w = huffWeight[n];
U32 const length = (1 << w) >> 1;
U32 i;
HUF_DEltX2 D;
D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w);
for (i = rankVal[w]; i < rankVal[w] + length; i++)
dt[i] = D;
rankVal[w] += length;
} }
return iSize;
}
static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */
BYTE const c = dt[val].byte;
BIT_skipBits(Dstream, dt[val].nbBits);
return c;
}
#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \
*ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr)
static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog)
{
BYTE* const pStart = p;
/* up to 4 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) {
HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
HUF_DECODE_SYMBOLX2_1(p, bitDPtr);
HUF_DECODE_SYMBOLX2_2(p, bitDPtr);
HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
}
/* closer to the end */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd))
HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
/* no more data to retrieve from bitstream, hence no need to reload */
while (p < pEnd)
HUF_DECODE_SYMBOLX2_0(p, bitDPtr);
return pEnd-pStart;
}
static size_t HUF_decompress1X2_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + dstSize;
const void* dtPtr = DTable + 1;
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
BIT_DStream_t bitD;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog;
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode; }
HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog);
/* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
return dstSize;
}
size_t HUF_decompress1X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC);
return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
}
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize);
}
static size_t HUF_decompress4X2_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
/* Check */
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable + 1;
const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
size_t const length1 = MEM_readLE16(istart);
size_t const length2 = MEM_readLE16(istart+2);
size_t const length3 = MEM_readLE16(istart+4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
const size_t segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) {
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_1(op1, &bitD1);
HUF_DECODE_SYMBOLX2_1(op2, &bitD2);
HUF_DECODE_SYMBOLX2_1(op3, &bitD3);
HUF_DECODE_SYMBOLX2_1(op4, &bitD4);
HUF_DECODE_SYMBOLX2_2(op1, &bitD1);
HUF_DECODE_SYMBOLX2_2(op2, &bitD2);
HUF_DECODE_SYMBOLX2_2(op3, &bitD3);
HUF_DECODE_SYMBOLX2_2(op4, &bitD4);
HUF_DECODE_SYMBOLX2_0(op1, &bitD1);
HUF_DECODE_SYMBOLX2_0(op2, &bitD2);
HUF_DECODE_SYMBOLX2_0(op3, &bitD3);
HUF_DECODE_SYMBOLX2_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 supposed already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog);
HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog);
HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog);
HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog);
/* check */
endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endSignal) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
}
size_t HUF_decompress4X2_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 0) return ERROR(GENERIC);
return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx);
}
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX);
return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
/* *************************/
/* double-symbols decoding */
/* *************************/
typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */
typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t;
static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed,
const U32* rankValOrigin, const int minWeight,
const sortedSymbol_t* sortedSymbols, const U32 sortedListSize,
U32 nbBitsBaseline, U16 baseSeq)
{
HUF_DEltX4 DElt;
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
/* get pre-calculated rankVal */
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill skipped values */
if (minWeight>1) {
U32 i, skipSize = rankVal[minWeight];
MEM_writeLE16(&(DElt.sequence), baseSeq);
DElt.nbBits = (BYTE)(consumed);
DElt.length = 1;
for (i = 0; i < skipSize; i++)
DTable[i] = DElt;
}
/* fill DTable */
{ U32 s; for (s=0; s<sortedListSize; s++) { /* note : sortedSymbols already skipped */
const U32 symbol = sortedSymbols[s].symbol;
const U32 weight = sortedSymbols[s].weight;
const U32 nbBits = nbBitsBaseline - weight;
const U32 length = 1 << (sizeLog-nbBits);
const U32 start = rankVal[weight];
U32 i = start;
const U32 end = start + length;
MEM_writeLE16(&(DElt.sequence), (U16)(baseSeq + (symbol << 8)));
DElt.nbBits = (BYTE)(nbBits + consumed);
DElt.length = 2;
do { DTable[i++] = DElt; } while (i<end); /* since length >= 1 */
rankVal[weight] += length;
} }
}
typedef U32 rankVal_t[HUF_TABLELOG_ABSOLUTEMAX][HUF_TABLELOG_ABSOLUTEMAX + 1];
static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog,
const sortedSymbol_t* sortedList, const U32 sortedListSize,
const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight,
const U32 nbBitsBaseline)
{
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1];
const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */
const U32 minBits = nbBitsBaseline - maxWeight;
U32 s;
memcpy(rankVal, rankValOrigin, sizeof(rankVal));
/* fill DTable */
for (s=0; s<sortedListSize; s++) {
const U16 symbol = sortedList[s].symbol;
const U32 weight = sortedList[s].weight;
const U32 nbBits = nbBitsBaseline - weight;
const U32 start = rankVal[weight];
const U32 length = 1 << (targetLog-nbBits);
if (targetLog-nbBits >= minBits) { /* enough room for a second symbol */
U32 sortedRank;
int minWeight = nbBits + scaleLog;
if (minWeight < 1) minWeight = 1;
sortedRank = rankStart[minWeight];
HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits,
rankValOrigin[nbBits], minWeight,
sortedList+sortedRank, sortedListSize-sortedRank,
nbBitsBaseline, symbol);
} else {
HUF_DEltX4 DElt;
MEM_writeLE16(&(DElt.sequence), symbol);
DElt.nbBits = (BYTE)(nbBits);
DElt.length = 1;
{ U32 const end = start + length;
U32 u;
for (u = start; u < end; u++) DTable[u] = DElt;
} }
rankVal[weight] += length;
}
}
size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUF_SYMBOLVALUE_MAX + 1];
sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1];
U32 rankStats[HUF_TABLELOG_ABSOLUTEMAX + 1] = { 0 };
U32 rankStart0[HUF_TABLELOG_ABSOLUTEMAX + 2] = { 0 };
U32* const rankStart = rankStart0+1;
rankVal_t rankVal;
U32 tableLog, maxW, sizeOfSort, nbSymbols;
DTableDesc dtd = HUF_getDTableDesc(DTable);
U32 const maxTableLog = dtd.maxTableLog;
size_t iSize;
void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */
HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr;
HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */
if (maxTableLog > HUF_TABLELOG_ABSOLUTEMAX) return ERROR(tableLog_tooLarge);
/* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */
iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize);
if (HUF_isError(iSize)) return iSize;
/* check result */
if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */
/* find maxWeight */
for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */
/* Get start index of each weight */
{ U32 w, nextRankStart = 0;
for (w=1; w<maxW+1; w++) {
U32 current = nextRankStart;
nextRankStart += rankStats[w];
rankStart[w] = current;
}
rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/
sizeOfSort = nextRankStart;
}
/* sort symbols by weight */
{ U32 s;
for (s=0; s<nbSymbols; s++) {
U32 const w = weightList[s];
U32 const r = rankStart[w]++;
sortedSymbol[r].symbol = (BYTE)s;
sortedSymbol[r].weight = (BYTE)w;
}
rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */
}
/* Build rankVal */
{ U32* const rankVal0 = rankVal[0];
{ int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */
U32 nextRankVal = 0;
U32 w;
for (w=1; w<maxW+1; w++) {
U32 current = nextRankVal;
nextRankVal += rankStats[w] << (w+rescale);
rankVal0[w] = current;
} }
{ U32 const minBits = tableLog+1 - maxW;
U32 consumed;
for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) {
U32* const rankValPtr = rankVal[consumed];
U32 w;
for (w = 1; w < maxW+1; w++) {
rankValPtr[w] = rankVal0[w] >> consumed;
} } } }
HUF_fillDTableX4(dt, maxTableLog,
sortedSymbol, sizeOfSort,
rankStart0, rankVal, maxW,
tableLog+1);
dtd.tableLog = (BYTE)maxTableLog;
dtd.tableType = 1;
memcpy(DTable, &dtd, sizeof(dtd));
return iSize;
}
static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 2);
BIT_skipBits(DStream, dt[val].nbBits);
return dt[val].length;
}
static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog)
{
size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */
memcpy(op, dt+val, 1);
if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits);
else {
if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) {
BIT_skipBits(DStream, dt[val].nbBits);
if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8))
DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */
} }
return 1;
}
#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \
if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \
if (MEM_64bits()) \
ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog)
static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog)
{
BYTE* const pStart = p;
/* up to 8 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) {
HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
HUF_DECODE_SYMBOLX4_1(p, bitDPtr);
HUF_DECODE_SYMBOLX4_2(p, bitDPtr);
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
}
/* closer to end : up to 2 symbols at a time */
while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2))
HUF_DECODE_SYMBOLX4_0(p, bitDPtr);
while (p <= pEnd-2)
HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */
if (p < pEnd)
p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog);
return p-pStart;
}
static size_t HUF_decompress1X4_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
BIT_DStream_t bitD;
/* Init */
{ size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize);
if (HUF_isError(errorCode)) return errorCode;
}
/* decode */
{ BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog);
}
/* check */
if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected);
/* decoded size */
return dstSize;
}
size_t HUF_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC);
return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx);
}
size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
static size_t HUF_decompress4X4_usingDTable_internal(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
{ const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
BYTE* const oend = ostart + dstSize;
const void* const dtPtr = DTable+1;
const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr;
/* Init */
BIT_DStream_t bitD1;
BIT_DStream_t bitD2;
BIT_DStream_t bitD3;
BIT_DStream_t bitD4;
size_t const length1 = MEM_readLE16(istart);
size_t const length2 = MEM_readLE16(istart+2);
size_t const length3 = MEM_readLE16(istart+4);
size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6);
const BYTE* const istart1 = istart + 6; /* jumpTable */
const BYTE* const istart2 = istart1 + length1;
const BYTE* const istart3 = istart2 + length2;
const BYTE* const istart4 = istart3 + length3;
size_t const segmentSize = (dstSize+3) / 4;
BYTE* const opStart2 = ostart + segmentSize;
BYTE* const opStart3 = opStart2 + segmentSize;
BYTE* const opStart4 = opStart3 + segmentSize;
BYTE* op1 = ostart;
BYTE* op2 = opStart2;
BYTE* op3 = opStart3;
BYTE* op4 = opStart4;
U32 endSignal;
DTableDesc const dtd = HUF_getDTableDesc(DTable);
U32 const dtLog = dtd.tableLog;
if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */
{ size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3);
if (HUF_isError(errorCode)) return errorCode; }
{ size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4);
if (HUF_isError(errorCode)) return errorCode; }
/* 16-32 symbols per loop (4-8 symbols per stream) */
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) {
HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
HUF_DECODE_SYMBOLX4_1(op1, &bitD1);
HUF_DECODE_SYMBOLX4_1(op2, &bitD2);
HUF_DECODE_SYMBOLX4_1(op3, &bitD3);
HUF_DECODE_SYMBOLX4_1(op4, &bitD4);
HUF_DECODE_SYMBOLX4_2(op1, &bitD1);
HUF_DECODE_SYMBOLX4_2(op2, &bitD2);
HUF_DECODE_SYMBOLX4_2(op3, &bitD3);
HUF_DECODE_SYMBOLX4_2(op4, &bitD4);
HUF_DECODE_SYMBOLX4_0(op1, &bitD1);
HUF_DECODE_SYMBOLX4_0(op2, &bitD2);
HUF_DECODE_SYMBOLX4_0(op3, &bitD3);
HUF_DECODE_SYMBOLX4_0(op4, &bitD4);
endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4);
}
/* check corruption */
if (op1 > opStart2) return ERROR(corruption_detected);
if (op2 > opStart3) return ERROR(corruption_detected);
if (op3 > opStart4) return ERROR(corruption_detected);
/* note : op4 already verified within main loop */
/* finish bitStreams one by one */
HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog);
HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog);
HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog);
HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog);
/* check */
{ U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4);
if (!endCheck) return ERROR(corruption_detected); }
/* decoded size */
return dstSize;
}
}
size_t HUF_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc dtd = HUF_getDTableDesc(DTable);
if (dtd.tableType != 1) return ERROR(GENERIC);
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
const BYTE* ip = (const BYTE*) cSrc;
size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize);
if (HUF_isError(hSize)) return hSize;
if (hSize >= cSrcSize) return ERROR(srcSize_wrong);
ip += hSize; cSrcSize -= hSize;
return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx);
}
size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX);
return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
/* ********************************/
/* Generic decompression selector */
/* ********************************/
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
}
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
const void* cSrc, size_t cSrcSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) :
HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable);
}
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
{
/* single, double, quad */
{{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */
{{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */
{{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */
{{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */
{{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */
{{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */
{{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */
{{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */
{{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */
{{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */
{{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */
{{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */
{{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */
{{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
* based on a set of pre-determined metrics.
* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 .
* Assumption : 0 < cSrcSize < dstSize <= 128 KB */
U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
{
/* decoder timing evaluation */
U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */
U32 const D256 = (U32)(dstSize >> 8);
U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256);
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */
return DTime1 < DTime0;
}
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 };
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
}
}
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
}
}
size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
}
}
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */
if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
}
}
C/lizard/lz5_common.h
+497
View File
@@ -0,0 +1,497 @@
/*
LZ5 - Fast LZ compression algorithm
Copyright (C) 2011-2015, Yann Collet.
Copyright (C) 2015-2016, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
#ifndef LZ5_COMMON_H_2983
#define LZ5_COMMON_H_2983
#if defined (__cplusplus)
extern "C" {
#endif
/*-************************************
* Memory routines
**************************************/
#include <stdlib.h> /* malloc, calloc, free */
#include <string.h> /* memset, memcpy */
#include <stdint.h> /* intptr_t */
#include "entropy/mem.h"
#include "lz5_compress.h" /* LZ5_GCC_VERSION */
//#define LZ5_USE_LOGS
#define LZ5_LOG_COMPRESS(...) //printf(__VA_ARGS__)
#define LZ5_LOG_DECOMPRESS(...) //printf(__VA_ARGS__)
#define LZ5_LOG_COMPRESS_LZ4(...) //printf(__VA_ARGS__)
#define COMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__)
#define LZ5_LOG_DECOMPRESS_LZ4(...) //printf(__VA_ARGS__)
#define DECOMPLOG_CODEWORDS_LZ4(...) //printf(__VA_ARGS__)
#define LZ5_LOG_COMPRESS_LZ5v2(...) //printf(__VA_ARGS__)
#define COMPLOG_CODEWORDS_LZ5v2(...) //printf(__VA_ARGS__)
#define LZ5_LOG_DECOMPRESS_LZ5v2(...) //printf(__VA_ARGS__)
#define DECOMPLOG_CODEWORDS_LZ5v2(...) //printf(__VA_ARGS__)
/*-************************************
* Common Constants
**************************************/
#define MINMATCH 4
//#define USE_LZ4_ONLY
//#define LZ5_USE_TEST
#define LZ5_DICT_SIZE (1<<24)
#define WILDCOPYLENGTH 16
#define LASTLITERALS WILDCOPYLENGTH
#define MFLIMIT (WILDCOPYLENGTH+MINMATCH)
#define LZ5_MAX_PRICE (1<<28)
#define LZ5_INIT_LAST_OFFSET 0
#define LZ5_MAX_16BIT_OFFSET (1<<16)
#define MM_LONGOFF 16
#define LZ5_BLOCK_SIZE_PAD (LZ5_BLOCK_SIZE+32)
#define LZ5_COMPRESS_ADD_BUF (5*LZ5_BLOCK_SIZE_PAD)
#ifndef LZ5_NO_HUFFMAN
#define LZ5_COMPRESS_ADD_HUF HUF_compressBound(LZ5_BLOCK_SIZE_PAD)
#define LZ5_HUF_BLOCK_SIZE LZ5_BLOCK_SIZE
#else
#define LZ5_COMPRESS_ADD_HUF 0
#define LZ5_HUF_BLOCK_SIZE 1
#endif
/* LZ4 codewords */
#define ML_BITS_LZ4 4
#define ML_MASK_LZ4 ((1U<<ML_BITS_LZ4)-1)
#define RUN_BITS_LZ4 (8-ML_BITS_LZ4)
#define RUN_MASK_LZ4 ((1U<<RUN_BITS_LZ4)-1)
/* LZ5v2 codewords */
#define ML_BITS_LZ5v2 4
#define RUN_BITS_LZ5v2 3
#define ML_RUN_BITS (ML_BITS_LZ5v2 + RUN_BITS_LZ5v2)
#define MAX_SHORT_LITLEN 7
#define MAX_SHORT_MATCHLEN 15
#define LZ5_LAST_LONG_OFF 31
/* header byte */
#define LZ5_FLAG_LITERALS 1
#define LZ5_FLAG_FLAGS 2
#define LZ5_FLAG_OFFSET16 4
#define LZ5_FLAG_OFFSET24 8
#define LZ5_FLAG_LEN 16
#define LZ5_FLAG_UNCOMPRESSED 128
/* stream numbers */
#define LZ5_STREAM_LITERALS 0
#define LZ5_STREAM_FLAGS 1
#define LZ5_STREAM_OFFSET16 2
#define LZ5_STREAM_OFFSET24 3
#define LZ5_STREAM_LEN 4
#define LZ5_STREAM_UNCOMPRESSED 5
typedef enum { LZ5_parser_fastSmall, LZ5_parser_fast, LZ5_parser_fastBig, LZ5_parser_noChain, LZ5_parser_hashChain, LZ5_parser_priceFast, LZ5_parser_lowestPrice, LZ5_parser_optimalPrice, LZ5_parser_optimalPriceBT } LZ5_parser_type; /* from faster to stronger */
typedef enum { LZ5_coderwords_LZ4, LZ5_coderwords_LZ5v2 } LZ5_decompress_type;
typedef struct
{
U32 windowLog; /* largest match distance : impact decompression buffer size */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 hashLog3; /* dispatch table : larger == more memory, faster*/
U32 searchNum; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */
U32 minMatchLongOff; /* min match size with offsets >= 1<<16 */
U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */
U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */
LZ5_parser_type parserType;
LZ5_decompress_type decompressType;
} LZ5_parameters;
struct LZ5_stream_s
{
const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */
const BYTE* dictBase; /* alternate base for extDict */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 allocatedMemory;
int compressionLevel;
LZ5_parameters params;
U32 hashTableSize;
U32 chainTableSize;
U32* chainTable;
U32* hashTable;
int last_off;
const BYTE* off24pos;
U32 huffType;
U32 comprStreamLen;
BYTE* huffBase;
BYTE* huffEnd;
BYTE* offset16Base;
BYTE* offset24Base;
BYTE* lenBase;
BYTE* literalsBase;
BYTE* flagsBase;
BYTE* offset16Ptr;
BYTE* offset24Ptr;
BYTE* lenPtr;
BYTE* literalsPtr;
BYTE* flagsPtr;
BYTE* offset16End;
BYTE* offset24End;
BYTE* lenEnd;
BYTE* literalsEnd;
BYTE* flagsEnd;
U32 flagFreq[256];
U32 litFreq[256];
U32 litSum, flagSum;
U32 litPriceSum, log2LitSum, log2FlagSum;
U32 cachedPrice;
U32 cachedLitLength;
const BYTE* cachedLiterals;
const BYTE* diffBase;
const BYTE* srcBase;
const BYTE* destBase;
};
struct LZ5_dstream_s
{
const BYTE* offset16Ptr;
const BYTE* offset24Ptr;
const BYTE* lenPtr;
const BYTE* literalsPtr;
const BYTE* flagsPtr;
const BYTE* offset16End;
const BYTE* offset24End;
const BYTE* lenEnd;
const BYTE* literalsEnd;
const BYTE* flagsEnd;
const BYTE* diffBase;
intptr_t last_off;
};
typedef struct LZ5_dstream_s LZ5_dstream_t;
/* *************************************
* HC Pre-defined compression levels
***************************************/
#define LZ5_WINDOWLOG_LZ4 16
#define LZ5_CHAINLOG_LZ4 LZ5_WINDOWLOG_LZ4
#define LZ5_HASHLOG_LZ4 18
#define LZ5_HASHLOG_LZ4SM 12
#define LZ5_WINDOWLOG_LZ5v2 22
#define LZ5_CHAINLOG_LZ5v2 LZ5_WINDOWLOG_LZ5v2
#define LZ5_HASHLOG_LZ5v2 18
static const LZ5_parameters LZ5_defaultParameters[LZ5_MAX_CLEVEL+1-LZ5_MIN_CLEVEL] =
{
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LZ5_WINDOWLOG_LZ4, 0, LZ5_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, LZ5_parser_fastSmall, LZ5_coderwords_LZ4 }, // level 10
{ LZ5_WINDOWLOG_LZ4, 0, LZ5_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LZ5_parser_fast, LZ5_coderwords_LZ4 }, // level 11
{ LZ5_WINDOWLOG_LZ4, 0, LZ5_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LZ5_parser_noChain, LZ5_coderwords_LZ4 }, // level 12
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 13
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 14
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 15
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 16
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 17
{ LZ5_WINDOWLOG_LZ4, LZ5_WINDOWLOG_LZ4+1, LZ5_HASHLOG_LZ4, 16, 16, 4, 0, 1<<10, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ4 }, // level 18
{ LZ5_WINDOWLOG_LZ4, LZ5_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ4 }, // level 19
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LZ5_WINDOWLOG_LZ5v2, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, LZ5_parser_fastBig, LZ5_coderwords_LZ5v2 }, // level 20
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 14, 13, 1, 5, MM_LONGOFF, 0, 0, LZ5_parser_priceFast, LZ5_coderwords_LZ5v2 }, // level 21
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, LZ5_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 0, 0, LZ5_parser_priceFast, LZ5_coderwords_LZ5v2 }, // level 22
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, LZ5_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 64, 0, LZ5_parser_lowestPrice, LZ5_coderwords_LZ5v2 }, // level 23
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 23, 16, 2, 5, MM_LONGOFF, 64, 0, LZ5_parser_lowestPrice, LZ5_coderwords_LZ5v2 }, // level 24
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 23, 16, 8, 4, MM_LONGOFF, 64, 0, LZ5_parser_lowestPrice, LZ5_coderwords_LZ5v2 }, // level 25
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 26
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 27
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2+1, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 28
{ 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 29
#ifndef LZ5_NO_HUFFMAN
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LZ5_WINDOWLOG_LZ4, 0, LZ5_HASHLOG_LZ4SM, 0, 0, 0, 0, 0, 0, LZ5_parser_fastSmall, LZ5_coderwords_LZ4 }, // level 30
{ LZ5_WINDOWLOG_LZ4, 0, LZ5_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LZ5_parser_fast, LZ5_coderwords_LZ4 }, // level 31
{ LZ5_WINDOWLOG_LZ4, 0, 14, 0, 0, 0, 0, 0, 0, LZ5_parser_noChain, LZ5_coderwords_LZ4 }, // level 32
{ LZ5_WINDOWLOG_LZ4, 0, LZ5_HASHLOG_LZ4, 0, 0, 0, 0, 0, 0, LZ5_parser_noChain, LZ5_coderwords_LZ4 }, // level 33
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 2, 5, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 34
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 4, 5, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 35
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 8, 5, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 36
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 16, 4, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 37
{ LZ5_WINDOWLOG_LZ4, LZ5_CHAINLOG_LZ4, LZ5_HASHLOG_LZ4, 0, 256, 4, 0, 0, 0, LZ5_parser_hashChain, LZ5_coderwords_LZ4 }, // level 38
{ LZ5_WINDOWLOG_LZ4, LZ5_WINDOWLOG_LZ4+1, 23, 16, 256, 4, 0, 1<<10, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ4 }, // level 39
/* windLog, contentLog, HashLog, H3, Snum, SL, MMLongOff, SuffL, FS, Parser function, Decompressor type */
{ LZ5_WINDOWLOG_LZ5v2, 0, 14, 0, 1, 5, MM_LONGOFF, 0, 0, LZ5_parser_fastBig, LZ5_coderwords_LZ5v2 }, // level 40
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 14, 13, 1, 5, MM_LONGOFF, 0, 0, LZ5_parser_priceFast, LZ5_coderwords_LZ5v2 }, // level 41
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, LZ5_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 0, 0, LZ5_parser_priceFast, LZ5_coderwords_LZ5v2 }, // level 42
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, LZ5_HASHLOG_LZ5v2, 13, 1, 5, MM_LONGOFF, 64, 0, LZ5_parser_lowestPrice, LZ5_coderwords_LZ5v2 }, // level 43
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 23, 16, 2, 5, MM_LONGOFF, 64, 0, LZ5_parser_lowestPrice, LZ5_coderwords_LZ5v2 }, // level 44
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 23, 16, 8, 4, MM_LONGOFF, 64, 0, LZ5_parser_lowestPrice, LZ5_coderwords_LZ5v2 }, // level 45
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2, 23, 16, 8, 4, MM_LONGOFF, 64, 0, LZ5_parser_optimalPrice, LZ5_coderwords_LZ5v2 }, // level 46
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2+1, 23, 16, 8, 4, MM_LONGOFF, 64, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 47
{ LZ5_WINDOWLOG_LZ5v2, LZ5_CHAINLOG_LZ5v2+1, 23, 16, 128, 4, MM_LONGOFF, 64, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 48
{ 24, 25, 23, 24, 1<<10, 4, MM_LONGOFF, 1<<10, 1, LZ5_parser_optimalPriceBT, LZ5_coderwords_LZ5v2 }, // level 49
#endif
// { 10, 10, 10, 0, 0, 4, 0, 0, 0, LZ5_fast }, // min values
// { 24, 24, 28, 24, 1<<24, 7, 0, 1<<24, 2, LZ5_optimal_price }, // max values
};
/*-************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h>
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
#else
# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# if defined(__GNUC__) || defined(__clang__)
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif /* _MSC_VER */
#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#if (LZ5_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
# define expect(expr,value) (expr)
#endif
#define likely(expr) expect((expr) != 0, 1)
#define unlikely(expr) expect((expr) != 0, 0)
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define ALLOCATOR(n,s) calloc(n,s)
#define FREEMEM free
#define MEM_INIT memset
#ifndef MAX
#define MAX(a,b) ((a)>(b))?(a):(b)
#endif
#ifndef MIN
#define MIN(a,b) ((a)<(b)?(a):(b))
#endif
#if MINMATCH == 3
#define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr)<<8)
#else
#define MEM_readMINMATCH(ptr) (U32)(MEM_read32(ptr))
#endif
/*-************************************
* Reading and writing into memory
**************************************/
#define STEPSIZE sizeof(size_t)
MEM_STATIC void LZ5_copy8(void* dst, const void* src)
{
memcpy(dst,src,8);
}
/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */
MEM_STATIC void LZ5_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
{
BYTE* d = (BYTE*)dstPtr;
const BYTE* s = (const BYTE*)srcPtr;
BYTE* const e = (BYTE*)dstEnd;
#if 0
const size_t l2 = 8 - (((size_t)d) & (sizeof(void*)-1));
LZ5_copy8(d,s); if (d>e-9) return;
d+=l2; s+=l2;
#endif /* join to align */
do { LZ5_copy8(d,s); d+=8; s+=8; } while (d<e);
}
MEM_STATIC void LZ5_wildCopy16(BYTE* dstPtr, const BYTE* srcPtr, BYTE* dstEnd)
{
do {
LZ5_copy8(dstPtr, srcPtr);
LZ5_copy8(dstPtr+8, srcPtr+8);
dstPtr += 16;
srcPtr += 16;
}
while (dstPtr < dstEnd);
}
/*
* LZ5_FORCE_SW_BITCOUNT
* Define this parameter if your target system or compiler does not support hardware bit count
*/
#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */
# define LZ5_FORCE_SW_BITCOUNT
#endif
/* **************************************
* Function body to include for inlining
****************************************/
MEM_STATIC U32 LZ5_highbit32(U32 val)
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
/*-************************************
* Common functions
**************************************/
MEM_STATIC unsigned LZ5_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian()) {
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ5_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif (defined(__clang__) || (LZ5_GCC_VERSION >= 304)) && !defined(LZ5_FORCE_SW_BITCOUNT)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
} else /* 32 bits */ {
# if defined(_MSC_VER) && !defined(LZ5_FORCE_SW_BITCOUNT)
unsigned long r;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif (defined(__clang__) || (LZ5_GCC_VERSION >= 304)) && !defined(LZ5_FORCE_SW_BITCOUNT)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
} else /* Big Endian CPU */ {
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ5_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif (defined(__clang__) || (LZ5_GCC_VERSION >= 304)) && !defined(LZ5_FORCE_SW_BITCOUNT)
return (__builtin_clzll((U64)val) >> 3);
# else
unsigned r;
if (!(val>>32)) { r=4; } else { r=0; val>>=32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
} else /* 32 bits */ {
# if defined(_MSC_VER) && !defined(LZ5_FORCE_SW_BITCOUNT)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif (defined(__clang__) || (LZ5_GCC_VERSION >= 304)) && !defined(LZ5_FORCE_SW_BITCOUNT)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC unsigned LZ5_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while (likely(pIn<pInLimit-(STEPSIZE-1))) {
size_t diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (!diff) { pIn+=STEPSIZE; pMatch+=STEPSIZE; continue; }
pIn += LZ5_NbCommonBytes(diff);
return (unsigned)(pIn - pStart);
}
if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (unsigned)(pIn - pStart);
}
/* alias to functions with compressionLevel=1 */
int LZ5_sizeofState_MinLevel(void);
int LZ5_compress_MinLevel(const char* source, char* dest, int sourceSize, int maxDestSize);
int LZ5_compress_extState_MinLevel (void* state, const char* source, char* dest, int inputSize, int maxDestSize);
LZ5_stream_t* LZ5_resetStream_MinLevel (LZ5_stream_t* streamPtr);
LZ5_stream_t* LZ5_createStream_MinLevel(void);
#if defined (__cplusplus)
}
#endif
#endif /* LZ5_COMMON_H_2983827168210 */
C/lizard/lz5_compress.c
+637
View File
@@ -0,0 +1,637 @@
/*
LZ5 - Fast LZ compression algorithm
Copyright (C) 2011-2015, Yann Collet.
Copyright (C) 2015-2016, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOTLZ5_hash4Ptr
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
/* *************************************
* Includes
***************************************/
#include "lz5_compress.h"
#include "lz5_common.h"
#include <stdio.h>
#include <stdint.h> // intptr_t
#ifndef USE_LZ4_ONLY
#ifdef LZ5_USE_TEST
#include "test/lz5_common_test.h"
#include "test/lz5_compress_test.h"
#else
#include "lz5_compress_lz5v2.h"
#endif
#endif
#include "lz5_compress_lz4.h"
#include "entropy/huf.h"
/* *************************************
* Local Macros
***************************************/
#define DELTANEXT(p) chainTable[(p) & contentMask]
#define LZ5_MINIMAL_HUFF_GAIN(comprSize) (comprSize + (comprSize/8) + 512)
#define LZ5_MINIMAL_BLOCK_GAIN(comprSize) (comprSize + (comprSize/32) + 512)
/*-************************************
* Local Utils
**************************************/
int LZ5_versionNumber (void) { return LZ5_VERSION_NUMBER; }
int LZ5_compressBound(int isize) { return LZ5_COMPRESSBOUND(isize); }
int LZ5_sizeofState_MinLevel() { return LZ5_sizeofState(LZ5_MIN_CLEVEL); }
/* *************************************
* Hash functions
***************************************/
#define HASH_UPDATE_LIMIT 8 /* equal to MEM_read64 */
static const U32 prime4bytes = 2654435761U;
static const U64 prime5bytes = 889523592379ULL;
static const U64 prime6bytes = 227718039650203ULL;
static const U64 prime7bytes = 58295818150454627ULL;
#if MINMATCH == 3
static const U32 prime3bytes = 506832829U;
static U32 LZ5_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; }
static size_t LZ5_hash3Ptr(const void* ptr, U32 h) { return LZ5_hash3(MEM_read32(ptr), h); }
#endif
static U32 LZ5_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
static size_t LZ5_hash4Ptr(const void* ptr, U32 h) { return LZ5_hash4(MEM_read32(ptr), h); }
static size_t LZ5_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; }
static size_t LZ5_hash5Ptr(const void* p, U32 h) { return LZ5_hash5(MEM_read64(p), h); }
static size_t LZ5_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
static size_t LZ5_hash6Ptr(const void* p, U32 h) { return LZ5_hash6(MEM_read64(p), h); }
static size_t LZ5_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; }
static size_t LZ5_hash7Ptr(const void* p, U32 h) { return LZ5_hash7(MEM_read64(p), h); }
static size_t LZ5_hashPtr(const void* p, U32 hBits, U32 mls)
{
switch(mls)
{
default:
case 4: return LZ5_hash4Ptr(p, hBits);
case 5: return LZ5_hash5Ptr(p, hBits);
case 6: return LZ5_hash6Ptr(p, hBits);
case 7: return LZ5_hash7Ptr(p, hBits);
}
}
/**************************************
* Internal functions
**************************************/
/** LZ5_count_2segments() :
* can count match length with `ip` & `match` in 2 different segments.
* convention : on reaching mEnd, match count continue starting from iStart
*/
static size_t LZ5_count_2segments(const BYTE* ip, const BYTE* match, const BYTE* iEnd, const BYTE* mEnd, const BYTE* iStart)
{
const BYTE* const vEnd = MIN( ip + (mEnd - match), iEnd);
size_t const matchLength = LZ5_count(ip, match, vEnd);
if (match + matchLength != mEnd) return matchLength;
return matchLength + LZ5_count(ip+matchLength, iStart, iEnd);
}
void LZ5_initBlock(LZ5_stream_t* ctx)
{
ctx->offset16Ptr = ctx->offset16Base;
ctx->offset24Ptr = ctx->offset24Base;
ctx->lenPtr = ctx->lenBase;
ctx->literalsPtr = ctx->literalsBase;
ctx->flagsPtr = ctx->flagsBase;
ctx->last_off = LZ5_INIT_LAST_OFFSET; /* reset last offset */
}
FORCE_INLINE int LZ5_writeStream(int useHuff, LZ5_stream_t* ctx, BYTE* streamPtr, uint32_t streamLen, BYTE** op, BYTE* oend)
{
if (useHuff && streamLen > 1024) {
#ifndef LZ5_NO_HUFFMAN
int useHuffBuf;
if (*op + 6 > oend) { LZ5_LOG_COMPRESS("*op[%p] + 6 > oend[%p]\n", *op, oend); return -1; }
useHuffBuf = ((size_t)(oend - (*op + 6)) < HUF_compressBound(streamLen));
if (useHuffBuf) {
if (streamLen > LZ5_BLOCK_SIZE) { LZ5_LOG_COMPRESS("streamLen[%d] > LZ5_BLOCK_SIZE\n", streamLen); return -1; }
ctx->comprStreamLen = (U32)HUF_compress(ctx->huffBase, ctx->huffEnd - ctx->huffBase, streamPtr, streamLen);
} else {
ctx->comprStreamLen = (U32)HUF_compress(*op + 6, oend - (*op + 6), streamPtr, streamLen);
}
if (!HUF_isError(ctx->comprStreamLen)) {
if (ctx->comprStreamLen > 0 && (LZ5_MINIMAL_HUFF_GAIN(ctx->comprStreamLen) < streamLen)) { /* compressible */
MEM_writeLE24(*op, streamLen);
MEM_writeLE24(*op+3, ctx->comprStreamLen);
if (useHuffBuf) {
if ((size_t)(oend - (*op + 6)) < ctx->comprStreamLen) { LZ5_LOG_COMPRESS("*op[%p] oend[%p] comprStreamLen[%d]\n", *op, oend, (int)ctx->comprStreamLen); return -1; }
memcpy(*op + 6, ctx->huffBase, ctx->comprStreamLen);
}
*op += ctx->comprStreamLen + 6;
LZ5_LOG_COMPRESS("HUF_compress streamLen=%d comprStreamLen=%d\n", (int)streamLen, (int)ctx->comprStreamLen);
return 1;
} else { LZ5_LOG_COMPRESS("HUF_compress ERROR comprStreamLen=%d streamLen=%d\n", (int)ctx->comprStreamLen, (int)streamLen); }
} else { LZ5_LOG_COMPRESS("HUF_compress ERROR %d: %s\n", (int)ctx->comprStreamLen, HUF_getErrorName(ctx->comprStreamLen)); }
#else
LZ5_LOG_COMPRESS("compiled with LZ5_NO_HUFFMAN\n");
(void)ctx;
return -1;
#endif
} else ctx->comprStreamLen = 0;
if (*op + 3 + streamLen > oend) { LZ5_LOG_COMPRESS("*op[%p] + 3 + streamLen[%d] > oend[%p]\n", *op, streamLen, oend); return -1; }
MEM_writeLE24(*op, streamLen);
*op += 3;
memcpy(*op, streamPtr, streamLen);
*op += streamLen;
LZ5_LOG_COMPRESS("Uncompressed streamLen=%d\n", (int)streamLen);
return 0;
}
int LZ5_writeBlock(LZ5_stream_t* ctx, const BYTE* ip, uint32_t inputSize, BYTE** op, BYTE* oend)
{
int res;
uint32_t flagsLen = (uint32_t)(ctx->flagsPtr - ctx->flagsBase);
uint32_t literalsLen = (uint32_t)(ctx->literalsPtr - ctx->literalsBase);
uint32_t lenLen = (uint32_t)(ctx->lenPtr - ctx->lenBase);
uint32_t offset16Len = (uint32_t)(ctx->offset16Ptr - ctx->offset16Base);
uint32_t offset24Len = (uint32_t)(ctx->offset24Ptr - ctx->offset24Base);
uint32_t sum = flagsLen + literalsLen + lenLen + offset16Len + offset24Len;
#ifdef LZ5_USE_LOGS
uint32_t comprFlagsLen, comprLiteralsLen;
#endif
BYTE* start = *op;
if ((literalsLen < WILDCOPYLENGTH) || (sum+5*3+1 > inputSize)) goto _write_uncompressed;
*start = 0;
*op += 1;
res = LZ5_writeStream(0, ctx, ctx->lenBase, lenLen, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LZ5_FLAG_LEN);
res = LZ5_writeStream(ctx->huffType&LZ5_FLAG_OFFSET16, ctx, ctx->offset16Base, offset16Len, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LZ5_FLAG_OFFSET16);
res = LZ5_writeStream(ctx->huffType&LZ5_FLAG_OFFSET24, ctx, ctx->offset24Base, offset24Len, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LZ5_FLAG_OFFSET24);
res = LZ5_writeStream(ctx->huffType&LZ5_FLAG_FLAGS, ctx, ctx->flagsBase, flagsLen, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LZ5_FLAG_FLAGS);
#ifdef LZ5_USE_LOGS
comprFlagsLen = ctx->comprStreamLen;
#endif
res = LZ5_writeStream(ctx->huffType&LZ5_FLAG_LITERALS, ctx, ctx->literalsBase, literalsLen, op, oend);
if (res < 0) goto _output_error; else *start += (BYTE)(res*LZ5_FLAG_LITERALS);
#ifdef LZ5_USE_LOGS
comprLiteralsLen = ctx->comprStreamLen;
sum = (int)(*op-start);
#endif
if (LZ5_MINIMAL_BLOCK_GAIN((uint32_t)(*op-start)) > inputSize) goto _write_uncompressed;
LZ5_LOG_COMPRESS("%d: total=%d block=%d flagsLen[%.2f%%]=%d comprFlagsLen[%.2f%%]=%d literalsLen[%.2f%%]=%d comprLiteralsLen[%.2f%%]=%d lenLen=%d offset16Len[%.2f%%]=%d offset24Len[%.2f%%]=%d\n", (int)(ip - ctx->srcBase),
(int)(*op - ctx->destBase), sum, (flagsLen*100.0)/sum, flagsLen, (comprFlagsLen*100.0)/sum, comprFlagsLen, (literalsLen*100.0)/sum, literalsLen, (comprLiteralsLen*100.0)/sum, comprLiteralsLen,
lenLen, (offset16Len*100.0)/sum, offset16Len, (offset24Len*100.0)/sum, offset24Len);
return 0;
_write_uncompressed:
LZ5_LOG_COMPRESS("%d: total=%d block=%d UNCOMPRESSED inputSize=%u outSize=%d\n", (int)(ip - ctx->srcBase),
(int)(*op - ctx->destBase), (int)(*op-start), inputSize, (int)(oend-start));
if ((uint32_t)(oend - start) < inputSize + 4) goto _output_error;
*start = LZ5_FLAG_UNCOMPRESSED;
*op = start + 1;
MEM_writeLE24(*op, inputSize);
*op += 3;
memcpy(*op, ip, inputSize);
*op += inputSize;
return 0;
_output_error:
LZ5_LOG_COMPRESS("LZ5_writeBlock ERROR size=%d/%d flagsLen=%d literalsLen=%d lenLen=%d offset16Len=%d offset24Len=%d\n", (int)(*op-start), (int)(oend-start), flagsLen, literalsLen, lenLen, offset16Len, offset24Len);
return 1;
}
FORCE_INLINE int LZ5_encodeSequence (
LZ5_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor,
size_t matchLength,
const BYTE* const match)
{
#ifdef USE_LZ4_ONLY
return LZ5_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match);
#else
if (ctx->params.decompressType == LZ5_coderwords_LZ4)
return LZ5_encodeSequence_LZ4(ctx, ip, anchor, matchLength, match);
return LZ5_encodeSequence_LZ5v2(ctx, ip, anchor, matchLength, match);
#endif
}
FORCE_INLINE int LZ5_encodeLastLiterals (
LZ5_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor)
{
LZ5_LOG_COMPRESS("LZ5_encodeLastLiterals LZ5_coderwords_LZ4=%d\n", ctx->params.decompressType == LZ5_coderwords_LZ4);
#ifdef USE_LZ4_ONLY
return LZ5_encodeLastLiterals_LZ4(ctx, ip, anchor);
#else
if (ctx->params.decompressType == LZ5_coderwords_LZ4)
return LZ5_encodeLastLiterals_LZ4(ctx, ip, anchor);
return LZ5_encodeLastLiterals_LZ5v2(ctx, ip, anchor);
#endif
}
/**************************************
* Include parsers
**************************************/
#include "lz5_parser_hashchain.h"
#include "lz5_parser_nochain.h"
#include "lz5_parser_fast.h"
#include "lz5_parser_fastsmall.h"
#include "lz5_parser_fastbig.h"
#ifndef USE_LZ4_ONLY
#include "lz5_parser_optimal.h"
#include "lz5_parser_lowestprice.h"
#include "lz5_parser_pricefast.h"
#endif
int LZ5_verifyCompressionLevel(int compressionLevel)
{
(void)LZ5_hashPtr;
(void)LZ5_wildCopy16;
if (compressionLevel > LZ5_MAX_CLEVEL) compressionLevel = LZ5_MAX_CLEVEL;
if (compressionLevel < LZ5_MIN_CLEVEL) compressionLevel = LZ5_DEFAULT_CLEVEL;
return compressionLevel;
}
int LZ5_sizeofState(int compressionLevel)
{
LZ5_parameters params;
U32 hashTableSize, chainTableSize;
compressionLevel = LZ5_verifyCompressionLevel(compressionLevel);
params = LZ5_defaultParameters[compressionLevel - LZ5_MIN_CLEVEL];
// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog)));
hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog)));
chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog));
return sizeof(LZ5_stream_t) + hashTableSize + chainTableSize + LZ5_COMPRESS_ADD_BUF + (int)LZ5_COMPRESS_ADD_HUF;
}
static void LZ5_init(LZ5_stream_t* ctx, const BYTE* start)
{
MEM_INIT((void*)ctx->hashTable, 0, ctx->hashTableSize);
MEM_INIT(ctx->chainTable, 0x01, ctx->chainTableSize);
// printf("memset hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx->hashTable, ((BYTE*)ctx->hashTable) + ctx->hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+ctx->chainTableSize);
ctx->nextToUpdate = LZ5_DICT_SIZE;
ctx->base = start - LZ5_DICT_SIZE;
ctx->end = start;
ctx->dictBase = start - LZ5_DICT_SIZE;
ctx->dictLimit = LZ5_DICT_SIZE;
ctx->lowLimit = LZ5_DICT_SIZE;
ctx->last_off = LZ5_INIT_LAST_OFFSET;
ctx->litSum = 0;
}
/* if ctx==NULL memory is allocated and returned as value */
LZ5_stream_t* LZ5_initStream(LZ5_stream_t* ctx, int compressionLevel)
{
LZ5_parameters params;
U32 hashTableSize, chainTableSize;
void *tempPtr;
compressionLevel = LZ5_verifyCompressionLevel(compressionLevel);
params = LZ5_defaultParameters[compressionLevel - LZ5_MIN_CLEVEL];
// hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog3)+((size_t)1 << params.hashLog)));
hashTableSize = (U32)(sizeof(U32)*(((size_t)1 << params.hashLog)));
chainTableSize = (U32)(sizeof(U32)*((size_t)1 << params.contentLog));
if (!ctx)
{
ctx = (LZ5_stream_t*)malloc(sizeof(LZ5_stream_t) + hashTableSize + chainTableSize + LZ5_COMPRESS_ADD_BUF + LZ5_COMPRESS_ADD_HUF);
if (!ctx) { printf("ERROR: Cannot allocate %d MB (compressionLevel=%d)\n", (int)(sizeof(LZ5_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel); return 0; }
LZ5_LOG_COMPRESS("Allocated %d MB (compressionLevel=%d)\n", (int)(sizeof(LZ5_stream_t) + hashTableSize + chainTableSize)>>20, compressionLevel);
ctx->allocatedMemory = sizeof(LZ5_stream_t) + hashTableSize + chainTableSize + LZ5_COMPRESS_ADD_BUF + (U32)LZ5_COMPRESS_ADD_HUF;
// printf("malloc from=%p to=%p hashTable=%p hashEnd=%p chainTable=%p chainEnd=%p\n", ctx, ((BYTE*)ctx)+sizeof(LZ5_stream_t) + hashTableSize + chainTableSize, ctx->hashTable, ((BYTE*)ctx->hashTable) + hashTableSize, ctx->chainTable, ((BYTE*)ctx->chainTable)+chainTableSize);
}
tempPtr = ctx;
ctx->hashTable = (U32*)(tempPtr) + sizeof(LZ5_stream_t)/4;
ctx->hashTableSize = hashTableSize;
ctx->chainTable = ctx->hashTable + hashTableSize/4;
ctx->chainTableSize = chainTableSize;
ctx->params = params;
ctx->compressionLevel = (unsigned)compressionLevel;
if (compressionLevel < 30)
ctx->huffType = 0;
else
ctx->huffType = LZ5_FLAG_LITERALS + LZ5_FLAG_FLAGS; // + LZ5_FLAG_OFFSET16 + LZ5_FLAG_OFFSET24;
ctx->literalsBase = (BYTE*)ctx->hashTable + ctx->hashTableSize + ctx->chainTableSize;
ctx->flagsBase = ctx->literalsEnd = ctx->literalsBase + LZ5_BLOCK_SIZE_PAD;
ctx->lenBase = ctx->flagsEnd = ctx->flagsBase + LZ5_BLOCK_SIZE_PAD;
ctx->offset16Base = ctx->lenEnd = ctx->lenBase + LZ5_BLOCK_SIZE_PAD;
ctx->offset24Base = ctx->offset16End = ctx->offset16Base + LZ5_BLOCK_SIZE_PAD;
ctx->huffBase = ctx->offset24End = ctx->offset24Base + LZ5_BLOCK_SIZE_PAD;
ctx->huffEnd = ctx->huffBase + LZ5_COMPRESS_ADD_HUF;
return ctx;
}
LZ5_stream_t* LZ5_createStream(int compressionLevel)
{
LZ5_stream_t* ctx = LZ5_initStream(NULL, compressionLevel);
// if (ctx) printf("LZ5_createStream ctx=%p ctx->compressionLevel=%d\n", ctx, ctx->compressionLevel);
return ctx;
}
/* initialization */
LZ5_stream_t* LZ5_resetStream(LZ5_stream_t* ctx, int compressionLevel)
{
size_t wanted = LZ5_sizeofState(compressionLevel);
// printf("LZ5_resetStream ctx=%p cLevel=%d have=%d wanted=%d min=%d\n", ctx, compressionLevel, (int)have, (int)wanted, (int)sizeof(LZ5_stream_t));
if (ctx->allocatedMemory < wanted)
{
// printf("REALLOC ctx=%p cLevel=%d have=%d wanted=%d\n", ctx, compressionLevel, (int)have, (int)wanted);
LZ5_freeStream(ctx);
ctx = LZ5_createStream(compressionLevel);
}
else
{
LZ5_initStream(ctx, compressionLevel);
}
if (ctx) ctx->base = NULL;
return ctx;
}
int LZ5_freeStream(LZ5_stream_t* ctx)
{
if (ctx) {
// printf("LZ5_freeStream ctx=%p ctx->compressionLevel=%d\n", ctx, ctx->compressionLevel);
free(ctx);
}
return 0;
}
int LZ5_loadDict(LZ5_stream_t* LZ5_streamPtr, const char* dictionary, int dictSize)
{
LZ5_stream_t* ctxPtr = (LZ5_stream_t*) LZ5_streamPtr;
if (dictSize > LZ5_DICT_SIZE) {
dictionary += dictSize - LZ5_DICT_SIZE;
dictSize = LZ5_DICT_SIZE;
}
LZ5_init (ctxPtr, (const BYTE*)dictionary);
if (dictSize >= HASH_UPDATE_LIMIT) LZ5_Insert (ctxPtr, (const BYTE*)dictionary + (dictSize - (HASH_UPDATE_LIMIT-1)));
ctxPtr->end = (const BYTE*)dictionary + dictSize;
return dictSize;
}
static void LZ5_setExternalDict(LZ5_stream_t* ctxPtr, const BYTE* newBlock)
{
if (ctxPtr->end >= ctxPtr->base + HASH_UPDATE_LIMIT) LZ5_Insert (ctxPtr, ctxPtr->end - (HASH_UPDATE_LIMIT-1)); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = newBlock - ctxPtr->dictLimit;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
}
/* dictionary saving */
int LZ5_saveDict (LZ5_stream_t* LZ5_streamPtr, char* safeBuffer, int dictSize)
{
LZ5_stream_t* const ctx = (LZ5_stream_t*)LZ5_streamPtr;
int const prefixSize = (int)(ctx->end - (ctx->base + ctx->dictLimit));
//printf("LZ5_saveDict dictSize=%d prefixSize=%d ctx->dictLimit=%d\n", dictSize, prefixSize, (int)ctx->dictLimit);
if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memmove(safeBuffer, ctx->end - dictSize, dictSize);
{ U32 const endIndex = (U32)(ctx->end - ctx->base);
ctx->end = (const BYTE*)safeBuffer + dictSize;
ctx->base = ctx->end - endIndex;
ctx->dictLimit = endIndex - dictSize;
ctx->lowLimit = endIndex - dictSize;
if (ctx->nextToUpdate < ctx->dictLimit) ctx->nextToUpdate = ctx->dictLimit;
}
//printf("2LZ5_saveDict dictSize=%d\n", dictSize);
return dictSize;
}
FORCE_INLINE int LZ5_compress_generic (
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize)
{
LZ5_stream_t* ctx = (LZ5_stream_t*) ctxvoid;
size_t dictSize = (size_t)(ctx->end - ctx->base) - ctx->dictLimit;
const BYTE* ip = (const BYTE*) source;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
int res;
(void)dictSize;
LZ5_LOG_COMPRESS("LZ5_compress_generic source=%p inputSize=%d dest=%p maxOutputSize=%d cLevel=%d dictBase=%p dictSize=%d\n", source, inputSize, dest, maxOutputSize, ctx->compressionLevel, ctx->dictBase, (int)dictSize);
*op++ = (BYTE)ctx->compressionLevel;
maxOutputSize--; // can be lower than 0
ctx->end += inputSize;
ctx->srcBase = ctx->off24pos = ip;
ctx->destBase = (BYTE*)dest;
while (inputSize > 0)
{
int inputPart = MIN(LZ5_BLOCK_SIZE, inputSize);
if (ctx->huffType) LZ5_rescaleFreqs(ctx);
LZ5_initBlock(ctx);
ctx->diffBase = ip;
switch(ctx->params.parserType)
{
default:
case LZ5_parser_fastSmall:
res = LZ5_compress_fastSmall(ctx, ip, ip+inputPart); break;
case LZ5_parser_fast:
res = LZ5_compress_fast(ctx, ip, ip+inputPart); break;
case LZ5_parser_noChain:
res = LZ5_compress_noChain(ctx, ip, ip+inputPart); break;
case LZ5_parser_hashChain:
res = LZ5_compress_hashChain(ctx, ip, ip+inputPart); break;
#ifndef USE_LZ4_ONLY
case LZ5_parser_fastBig:
res = LZ5_compress_fastBig(ctx, ip, ip+inputPart); break;
case LZ5_parser_priceFast:
res = LZ5_compress_priceFast(ctx, ip, ip+inputPart); break;
case LZ5_parser_lowestPrice:
res = LZ5_compress_lowestPrice(ctx, ip, ip+inputPart); break;
case LZ5_parser_optimalPrice:
case LZ5_parser_optimalPriceBT:
res = LZ5_compress_optimalPrice(ctx, ip, ip+inputPart); break;
#else
case LZ5_parser_priceFast:
case LZ5_parser_lowestPrice:
case LZ5_parser_optimalPrice:
case LZ5_parser_optimalPriceBT:
res = 0;
#endif
}
LZ5_LOG_COMPRESS("LZ5_compress_generic res=%d inputPart=%d \n", res, inputPart);
if (res <= 0) return res;
if (LZ5_writeBlock(ctx, ip, inputPart, &op, oend)) goto _output_error;
ip += inputPart;
inputSize -= inputPart;
LZ5_LOG_COMPRESS("LZ5_compress_generic in=%d out=%d\n", (int)(ip-(const BYTE*)source), (int)(op-(BYTE*)dest));
}
LZ5_LOG_COMPRESS("LZ5_compress_generic total=%d\n", (int)(op-(BYTE*)dest));
return (int)(op-(BYTE*)dest);
_output_error:
LZ5_LOG_COMPRESS("LZ5_compress_generic ERROR\n");
return 0;
}
int LZ5_compress_continue (LZ5_stream_t* ctxPtr,
const char* source, char* dest,
int inputSize, int maxOutputSize)
{
/* auto-init if forgotten */
if (ctxPtr->base == NULL) LZ5_init (ctxPtr, (const BYTE*) source);
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) {
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE;
LZ5_loadDict((LZ5_stream_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
/* Check if blocks follow each other */
if ((const BYTE*)source != ctxPtr->end)
LZ5_setExternalDict(ctxPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */
{ const BYTE* sourceEnd = (const BYTE*) source + inputSize;
const BYTE* const dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* const dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) {
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
return LZ5_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize);
}
int LZ5_compress_extState (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
LZ5_stream_t* ctx = (LZ5_stream_t*) state;
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
/* initialize stream */
LZ5_initStream(ctx, compressionLevel);
LZ5_init ((LZ5_stream_t*)state, (const BYTE*)src);
return LZ5_compress_generic (state, src, dst, srcSize, maxDstSize);
}
int LZ5_compress(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
int cSize;
LZ5_stream_t* statePtr = LZ5_createStream(compressionLevel);
if (!statePtr) return 0;
cSize = LZ5_compress_extState(statePtr, src, dst, srcSize, maxDstSize, compressionLevel);
LZ5_freeStream(statePtr);
return cSize;
}
/**************************************
* Level1 functions
**************************************/
int LZ5_compress_extState_MinLevel(void* state, const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ5_compress_extState(state, source, dest, inputSize, maxOutputSize, LZ5_MIN_CLEVEL);
}
int LZ5_compress_MinLevel(const char* source, char* dest, int inputSize, int maxOutputSize)
{
return LZ5_compress(source, dest, inputSize, maxOutputSize, LZ5_MIN_CLEVEL);
}
LZ5_stream_t* LZ5_createStream_MinLevel(void)
{
return LZ5_createStream(LZ5_MIN_CLEVEL);
}
LZ5_stream_t* LZ5_resetStream_MinLevel(LZ5_stream_t* LZ5_stream)
{
return LZ5_resetStream (LZ5_stream, LZ5_MIN_CLEVEL);
}
C/lizard/lz5_compress.h
+208
View File
@@ -0,0 +1,208 @@
/*
LZ5 - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2016, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
#ifndef LZ5_H_2983
#define LZ5_H_2983
#if defined (__cplusplus)
extern "C" {
#endif
/*
* lz5_compress.h provides block compression functions. It gives full buffer control to user.
* Block compression functions are not-enough to send information,
* since it's still necessary to provide metadata (such as compressed size),
* and each application can do it in whichever way it wants.
* For interoperability, there is LZ5 frame specification (lz5_Frame_format.md).
* A library is provided to take care of it, see lz5frame.h.
*/
/*^***************************************************************
* Export parameters
*****************************************************************/
/*
* LZ5_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(LZ5_DLL_EXPORT) && (LZ5_DLL_EXPORT==1)
# define LZ5LIB_API __declspec(dllexport)
#elif defined(LZ5_DLL_IMPORT) && (LZ5_DLL_IMPORT==1)
# define LZ5LIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define LZ5LIB_API
#endif
/*-************************************
* Version
**************************************/
#define LZ5_VERSION_MAJOR 2 /* for breaking interface changes */
#define LZ5_VERSION_MINOR 0 /* for new (non-breaking) interface capabilities */
#define LZ5_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ5_VERSION_NUMBER (LZ5_VERSION_MAJOR *100*100 + LZ5_VERSION_MINOR *100 + LZ5_VERSION_RELEASE)
int LZ5_versionNumber (void);
#define LZ5_LIB_VERSION LZ5_VERSION_MAJOR.LZ5_VERSION_MINOR.LZ5_VERSION_RELEASE
#define LZ5_QUOTE(str) #str
#define LZ5_EXPAND_AND_QUOTE(str) LZ5_QUOTE(str)
#define LZ5_VERSION_STRING LZ5_EXPAND_AND_QUOTE(LZ5_LIB_VERSION)
const char* LZ5_versionString (void);
typedef struct LZ5_stream_s LZ5_stream_t;
#define LZ5_MIN_CLEVEL 10 /* minimum compression level */
#ifndef LZ5_NO_HUFFMAN
#define LZ5_MAX_CLEVEL 49 /* maximum compression level */
#else
#define LZ5_MAX_CLEVEL 29 /* maximum compression level */
#endif
#define LZ5_DEFAULT_CLEVEL 17
/*-************************************
* Simple Functions
**************************************/
LZ5LIB_API int LZ5_compress (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
LZ5_compress() :
Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= LZ5_compressBound(sourceSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ5_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails
*/
/*-************************************
* Advanced Functions
**************************************/
#define LZ5_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ5_BLOCK_SIZE (1<<17)
#define LZ5_BLOCK64K_SIZE (1<<16)
#define LZ5_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ5_MAX_INPUT_SIZE ? 0 : (isize) + 1 + 1 + ((isize/LZ5_BLOCK_SIZE)+1)*4)
/*!
LZ5_compressBound() :
Provides the maximum size that LZ5 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ5_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that LZ5_compress() compress faster when dest buffer size is >= LZ5_compressBound(srcSize)
inputSize : max supported value is LZ5_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ5_MAX_INPUT_SIZE)
*/
LZ5LIB_API int LZ5_compressBound(int inputSize);
/*!
LZ5_compress_extState() :
Same compression function, just using an externally allocated memory space to store compression state.
Use LZ5_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
*/
LZ5LIB_API int LZ5_sizeofState(int compressionLevel);
LZ5LIB_API int LZ5_compress_extState(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*-*********************************************
* Streaming Compression Functions
***********************************************/
/*! LZ5_createStream() will allocate and initialize an `LZ5_stream_t` structure.
* LZ5_freeStream() releases its memory.
* In the context of a DLL (liblz5), please use these methods rather than the static struct.
* They are more future proof, in case of a change of `LZ5_stream_t` size.
*/
LZ5LIB_API LZ5_stream_t* LZ5_createStream(int compressionLevel);
LZ5LIB_API int LZ5_freeStream (LZ5_stream_t* streamPtr);
/*! LZ5_resetStream() :
* Use this function to reset/reuse an allocated `LZ5_stream_t` structure
*/
LZ5LIB_API LZ5_stream_t* LZ5_resetStream (LZ5_stream_t* streamPtr, int compressionLevel);
/*! LZ5_loadDict() :
* Use this function to load a static dictionary into LZ5_stream.
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
* Loading a size of 0 is allowed.
* Return : dictionary size, in bytes (necessarily <= LZ5_DICT_SIZE)
*/
LZ5LIB_API int LZ5_loadDict (LZ5_stream_t* streamPtr, const char* dictionary, int dictSize);
/*! LZ5_compress_continue() :
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
* Important : Previous data blocks are assumed to still be present and unmodified !
* 'dst' buffer must be already allocated.
* If maxDstSize >= LZ5_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
*/
LZ5LIB_API int LZ5_compress_continue (LZ5_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize);
/*! LZ5_saveDict() :
* If previously compressed data block is not guaranteed to remain available at its memory location,
* save it into a safer place (char* safeBuffer).
* Note : you don't need to call LZ5_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call LZ5_compress_continue().
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error.
*/
LZ5LIB_API int LZ5_saveDict (LZ5_stream_t* streamPtr, char* safeBuffer, int dictSize);
#if defined (__cplusplus)
}
#endif
#endif /* LZ5_H_2983827168210 */
C/lizard/lz5_compress_lz4.h
+165
View File
@@ -0,0 +1,165 @@
#define LZ5_LENGTH_SIZE_LZ4(len) ((len >= (1<<16)+RUN_MASK_LZ4) ? 5 : ((len >= 254+RUN_MASK_LZ4) ? 3 : ((len >= RUN_MASK_LZ4) ? 1 : 0)))
FORCE_INLINE int LZ5_encodeSequence_LZ4 (
LZ5_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor,
size_t matchLength,
const BYTE* const match)
{
size_t length = (size_t)(*ip - *anchor);
BYTE* token = (ctx->flagsPtr)++;
(void) ctx;
COMPLOG_CODEWORDS_LZ4("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
/* Encode Literal length */
// if (ctx->literalsPtr > ctx->literalsEnd - length - LZ5_LENGTH_SIZE_LZ4(length) - 2 - WILDCOPYLENGTH) { LZ5_LOG_COMPRESS_LZ4("encodeSequence overflow1\n"); return 1; } /* Check output limit */
if (length >= RUN_MASK_LZ4)
{ size_t len = length - RUN_MASK_LZ4;
*token = RUN_MASK_LZ4;
if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; }
else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)len;
}
else *token = (BYTE)length;
/* Copy Literals */
if (length > 0) {
LZ5_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length);
#if 0 //def LZ5_USE_HUFFMAN
ctx->litSum += (U32)length;
ctx->litPriceSum += (U32)(length * ctx->log2LitSum);
{ U32 u;
for (u=0; u < length; u++) {
ctx->litPriceSum -= LZ5_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1);
ctx->litFreq[ctx->literalsPtr[u]]++;
} }
#endif
ctx->literalsPtr += length;
}
/* Encode Offset */
// if (match > *ip) printf("match > *ip\n"), exit(1);
// if ((U32)(*ip-match) >= (1<<16)) printf("off=%d\n", (U32)(*ip-match)), exit(1);
MEM_writeLE16(ctx->literalsPtr, (U16)(*ip-match));
ctx->literalsPtr+=2;
/* Encode MatchLength */
length = matchLength - MINMATCH;
// if (ctx->literalsPtr > ctx->literalsEnd - 5 /*LZ5_LENGTH_SIZE_LZ4(length)*/) { LZ5_LOG_COMPRESS_LZ4("encodeSequence overflow2\n"); return 1; } /* Check output limit */
if (length >= ML_MASK_LZ4) {
*token += (BYTE)(ML_MASK_LZ4<<RUN_BITS_LZ4);
length -= ML_MASK_LZ4;
if (length >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; }
else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)length;
}
else *token += (BYTE)(length<<RUN_BITS_LZ4);
#ifndef LZ5_NO_HUFFMAN
if (ctx->huffType) {
ctx->flagFreq[*token]++;
ctx->flagSum++;
LZ5_setLog2Prices(ctx);
}
#endif
/* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
return 0;
}
FORCE_INLINE int LZ5_encodeLastLiterals_LZ4 (
LZ5_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor)
{
size_t length = (int)(*ip - *anchor);
(void)ctx;
memcpy(ctx->literalsPtr, *anchor, length);
ctx->literalsPtr += length;
return 0;
}
#define LZ5_GET_TOKEN_PRICE_LZ4(token) (ctx->log2FlagSum - LZ5_highbit32(ctx->flagFreq[token]+1))
FORCE_INLINE size_t LZ5_get_price_LZ4(LZ5_stream_t* const ctx, const BYTE *ip, const size_t litLength, U32 offset, size_t matchLength)
{
size_t price = 0;
BYTE token = 0;
#if 0 //def LZ5_USE_HUFFMAN
const BYTE* literals = ip - litLength;
U32 u;
if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) {
size_t const additional = litLength - ctx->cachedLitLength;
// printf("%d ", (int)litLength - (int)ctx->cachedLitLength);
const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength;
price = ctx->cachedPrice + additional * ctx->log2LitSum;
for (u=0; u < additional; u++)
price -= LZ5_highbit32(ctx->litFreq[literals2[u]]+1);
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
} else {
price = litLength * ctx->log2LitSum;
for (u=0; u < litLength; u++)
price -= LZ5_highbit32(ctx->litFreq[literals[u]]+1);
if (litLength >= 12) {
ctx->cachedLiterals = literals;
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
}
}
#else
price += 8*litLength; /* Copy Literals */
(void)ip;
(void)ctx;
#endif
/* Encode Literal length */
if (litLength >= RUN_MASK_LZ4) {
size_t len = litLength - RUN_MASK_LZ4;
token = RUN_MASK_LZ4;
if (len >= (1<<16)) price += 32;
else if (len >= 254) price += 24;
else price += 8;
}
else token = (BYTE)litLength;
/* Encode MatchLength */
if (offset) {
size_t length;
price += 16; /* Encode Offset */
if (offset < 8) return LZ5_MAX_PRICE; // error
if (matchLength < MINMATCH) return LZ5_MAX_PRICE; // error
length = matchLength - MINMATCH;
if (length >= ML_MASK_LZ4) {
token += (BYTE)(ML_MASK_LZ4<<RUN_BITS_LZ4);
length -= ML_MASK_LZ4;
if (length >= (1<<16)) price += 32;
else if (length >= 254) price += 24;
else price += 8;
}
else token += (BYTE)(length<<RUN_BITS_LZ4);
}
if (ctx->huffType) {
if (offset > 0 || matchLength > 0) price += 2;
price += LZ5_GET_TOKEN_PRICE_LZ4(token);
} else {
price += 8; // token
}
return price;
}
C/lizard/lz5_compress_lz5v2.h
+301
View File
@@ -0,0 +1,301 @@
#define LZ5_FREQ_DIV 5
FORCE_INLINE void LZ5_setLog2Prices(LZ5_stream_t* ctx)
{
ctx->log2LitSum = LZ5_highbit32(ctx->litSum+1);
ctx->log2FlagSum = LZ5_highbit32(ctx->flagSum+1);
}
MEM_STATIC void LZ5_rescaleFreqs(LZ5_stream_t* ctx)
{
unsigned u;
ctx->cachedLiterals = NULL;
ctx->cachedPrice = ctx->cachedLitLength = 0;
ctx->litPriceSum = 0;
if (ctx->litSum == 0) {
ctx->litSum = 2 * 256;
ctx->flagSum = 2 * 256;
for (u=0; u < 256; u++) {
ctx->litFreq[u] = 2;
ctx->flagFreq[u] = 2;
}
} else {
ctx->litSum = 0;
ctx->flagSum = 0;
for (u=0; u < 256; u++) {
ctx->litFreq[u] = 1 + (ctx->litFreq[u]>>LZ5_FREQ_DIV);
ctx->litSum += ctx->litFreq[u];
ctx->flagFreq[u] = 1 + (ctx->flagFreq[u]>>LZ5_FREQ_DIV);
ctx->flagSum += ctx->flagFreq[u];
}
}
LZ5_setLog2Prices(ctx);
}
FORCE_INLINE int LZ5_encodeSequence_LZ5v2 (
LZ5_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor,
size_t matchLength,
const BYTE* const match)
{
U32 offset = (U32)(*ip - match);
size_t length = (size_t)(*ip - *anchor);
BYTE* token = (ctx->flagsPtr)++;
if (length > 0 || offset < LZ5_MAX_16BIT_OFFSET) {
/* Encode Literal length */
// if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - length - LZ5_LENGTH_SIZE_LZ5v2(length) - WILDCOPYLENGTH)) { LZ5_LOG_COMPRESS_LZ5v2("encodeSequence overflow1\n"); return 1; } /* Check output limit */
if (length >= MAX_SHORT_LITLEN)
{ size_t len;
*token = MAX_SHORT_LITLEN;
len = length - MAX_SHORT_LITLEN;
if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; }
else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)len;
}
else *token = (BYTE)length;
/* Copy Literals */
LZ5_wildCopy(ctx->literalsPtr, *anchor, (ctx->literalsPtr) + length);
#ifndef LZ5_NO_HUFFMAN
if (ctx->huffType) {
ctx->litSum += (U32)length;
ctx->litPriceSum += (U32)(length * ctx->log2LitSum);
{ U32 u;
for (u=0; u < length; u++) {
ctx->litPriceSum -= LZ5_highbit32(ctx->litFreq[ctx->literalsPtr[u]]+1);
ctx->litFreq[ctx->literalsPtr[u]]++;
} }
}
#endif
ctx->literalsPtr += length;
if (offset >= LZ5_MAX_16BIT_OFFSET) {
COMPLOG_CODEWORDS_LZ5v2("T32+ literal=%u match=%u offset=%d\n", (U32)length, 0, 0);
*token+=(1<<ML_RUN_BITS);
#ifndef LZ5_NO_HUFFMAN
if (ctx->huffType) {
ctx->flagFreq[*token]++;
ctx->flagSum++;
}
#endif
token = (ctx->flagsPtr)++;
}
}
/* Encode Offset */
if (offset >= LZ5_MAX_16BIT_OFFSET) // 24-bit offset
{
if (matchLength < MM_LONGOFF) printf("ERROR matchLength=%d/%d\n", (int)matchLength, MM_LONGOFF), exit(0);
// if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 8 /*LZ5_LENGTH_SIZE_LZ5v2(length)*/)) { LZ5_LOG_COMPRESS_LZ5v2("encodeSequence overflow2\n"); return 1; } /* Check output limit */
if (matchLength - MM_LONGOFF >= LZ5_LAST_LONG_OFF)
{
size_t len = matchLength - MM_LONGOFF - LZ5_LAST_LONG_OFF;
*token = LZ5_LAST_LONG_OFF;
if (len >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(len)); ctx->literalsPtr += 4; }
else if (len >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(len)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)len;
COMPLOG_CODEWORDS_LZ5v2("T31 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset);
}
else
{
COMPLOG_CODEWORDS_LZ5v2("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)matchLength, offset);
*token = (BYTE)(matchLength - MM_LONGOFF);
}
MEM_writeLE24(ctx->offset24Ptr, offset);
ctx->offset24Ptr += 3;
ctx->last_off = offset;
ctx->off24pos = *ip;
}
else
{
COMPLOG_CODEWORDS_LZ5v2("T32+ literal=%u match=%u offset=%d\n", (U32)length, (U32)matchLength, offset);
if (offset == 0)
{
*token+=(1<<ML_RUN_BITS);
}
else
{
if (offset < 8) printf("ERROR offset=%d\n", (int)offset);
if (matchLength < MINMATCH) { printf("matchLength[%d] < MINMATCH offset=%d\n", (int)matchLength, (int)ctx->last_off); exit(1); }
ctx->last_off = offset;
MEM_writeLE16(ctx->offset16Ptr, (U16)ctx->last_off); ctx->offset16Ptr += 2;
}
/* Encode MatchLength */
length = matchLength;
// if ((limitedOutputBuffer) && (ctx->literalsPtr > oend - 5 /*LZ5_LENGTH_SIZE_LZ5v2(length)*/)) { LZ5_LOG_COMPRESS_LZ5v2("encodeSequence overflow2\n"); return 1; } /* Check output limit */
if (length >= MAX_SHORT_MATCHLEN) {
*token += (BYTE)(MAX_SHORT_MATCHLEN<<RUN_BITS_LZ5v2);
length -= MAX_SHORT_MATCHLEN;
if (length >= (1<<16)) { *(ctx->literalsPtr) = 255; MEM_writeLE24(ctx->literalsPtr+1, (U32)(length)); ctx->literalsPtr += 4; }
else if (length >= 254) { *(ctx->literalsPtr) = 254; MEM_writeLE16(ctx->literalsPtr+1, (U16)(length)); ctx->literalsPtr += 3; }
else *(ctx->literalsPtr)++ = (BYTE)length;
}
else *token += (BYTE)(length<<RUN_BITS_LZ5v2);
}
#ifndef LZ5_NO_HUFFMAN
if (ctx->huffType) {
ctx->flagFreq[*token]++;
ctx->flagSum++;
LZ5_setLog2Prices(ctx);
}
#endif
/* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
return 0;
}
FORCE_INLINE int LZ5_encodeLastLiterals_LZ5v2 (
LZ5_stream_t* ctx,
const BYTE** ip,
const BYTE** anchor)
{
size_t length = (int)(*ip - *anchor);
(void)ctx;
memcpy(ctx->literalsPtr, *anchor, length);
ctx->literalsPtr += length;
return 0;
}
#define LZ5_PRICE_MULT 1
#define LZ5_GET_TOKEN_PRICE_LZ5v2(token) (LZ5_PRICE_MULT * (ctx->log2FlagSum - LZ5_highbit32(ctx->flagFreq[token]+1)))
FORCE_INLINE size_t LZ5_get_price_LZ5v2(LZ5_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength)
{
size_t price = 0;
BYTE token = 0;
#ifndef LZ5_NO_HUFFMAN
const BYTE* literals = ip - litLength;
U32 u;
if ((ctx->huffType) && (ctx->params.parserType != LZ5_parser_lowestPrice)) {
if (ctx->cachedLiterals == literals && litLength >= ctx->cachedLitLength) {
size_t const additional = litLength - ctx->cachedLitLength;
// printf("%d ", (int)litLength - (int)ctx->cachedLitLength);
const BYTE* literals2 = ctx->cachedLiterals + ctx->cachedLitLength;
price = ctx->cachedPrice + LZ5_PRICE_MULT * additional * ctx->log2LitSum;
for (u=0; u < additional; u++)
price -= LZ5_PRICE_MULT * LZ5_highbit32(ctx->litFreq[literals2[u]]+1);
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
} else {
price = LZ5_PRICE_MULT * litLength * ctx->log2LitSum;
for (u=0; u < litLength; u++)
price -= LZ5_PRICE_MULT * LZ5_highbit32(ctx->litFreq[literals[u]]+1);
if (litLength >= 12) {
ctx->cachedLiterals = literals;
ctx->cachedPrice = (U32)price;
ctx->cachedLitLength = (U32)litLength;
}
}
}
else
price += 8*litLength; /* Copy Literals */
#else
price += 8*litLength; /* Copy Literals */
(void)ip;
(void)ctx;
#endif
(void)off24pos;
(void)rep;
if (litLength > 0 || offset < LZ5_MAX_16BIT_OFFSET) {
/* Encode Literal length */
if (litLength >= MAX_SHORT_LITLEN)
{ size_t len = litLength - MAX_SHORT_LITLEN;
token = MAX_SHORT_LITLEN;
if (len >= (1<<16)) price += 32;
else if (len >= 254) price += 24;
else price += 8;
}
else token = (BYTE)litLength;
if (offset >= LZ5_MAX_16BIT_OFFSET) {
token+=(1<<ML_RUN_BITS);
if (ctx->huffType && ctx->params.parserType != LZ5_parser_lowestPrice)
price += LZ5_GET_TOKEN_PRICE_LZ5v2(token);
else
price += 8;
}
}
/* Encode Offset */
if (offset >= LZ5_MAX_16BIT_OFFSET) { // 24-bit offset
if (matchLength < MM_LONGOFF) return LZ5_MAX_PRICE; // error
if (matchLength - MM_LONGOFF >= LZ5_LAST_LONG_OFF) {
size_t len = matchLength - MM_LONGOFF - LZ5_LAST_LONG_OFF;
token = LZ5_LAST_LONG_OFF;
if (len >= (1<<16)) price += 32;
else if (len >= 254) price += 24;
else price += 8;
} else {
token = (BYTE)(matchLength - MM_LONGOFF);
}
price += 24;
} else {
size_t length;
if (offset == 0) {
token+=(1<<ML_RUN_BITS);
} else {
if (offset < 8) return LZ5_MAX_PRICE; // error
if (matchLength < MINMATCH) return LZ5_MAX_PRICE; // error
price += 16;
}
/* Encode MatchLength */
length = matchLength;
if (length >= MAX_SHORT_MATCHLEN) {
token += (BYTE)(MAX_SHORT_MATCHLEN<<RUN_BITS_LZ5v2);
length -= MAX_SHORT_MATCHLEN;
if (length >= (1<<16)) price += 32;
else if (length >= 254) price += 24;
else price += 8;
}
else token += (BYTE)(length<<RUN_BITS_LZ5v2);
}
if (offset > 0 || matchLength > 0) {
int offset_load = LZ5_highbit32(offset);
if (ctx->huffType) {
price += ((offset_load>=20) ? ((offset_load-19)*4) : 0);
price += 4 + (matchLength==1);
} else {
price += ((offset_load>=16) ? ((offset_load-15)*4) : 0);
price += 6 + (matchLength==1);
}
if (ctx->huffType && ctx->params.parserType != LZ5_parser_lowestPrice)
price += LZ5_GET_TOKEN_PRICE_LZ5v2(token);
else
price += 8;
} else {
if (ctx->huffType && ctx->params.parserType != LZ5_parser_lowestPrice)
price += LZ5_GET_TOKEN_PRICE_LZ5v2(token); // 1=better ratio
}
return price;
}
C/lizard/lz5_decompress.c
+375
View File
@@ -0,0 +1,375 @@
/*
LZ5 - Fast LZ compression algorithm
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2016, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
/**************************************
* Includes
**************************************/
//#define LZ5_STATS 1 // 0=simple stats, 1=more, 2=full
#ifdef LZ5_STATS
#include "test/lz5_stats.h"
#endif
#include "lz5_compress.h"
#include "lz5_decompress.h"
#include "lz5_common.h"
#include <stdio.h> // printf
#include <stdint.h> // intptr_t
/*-************************************
* Local Structures and types
**************************************/
typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive;
typedef enum { full = 0, partial = 1 } earlyEnd_directive;
#include "lz5_decompress_lz4.h"
#ifndef USE_LZ4_ONLY
#ifdef LZ5_USE_TEST
#include "test/lz5_common_test.h"
#include "test/lz5_decompress_test.h"
#else
#include "lz5_decompress_lz5v2.h"
#endif
#endif
#include "entropy/huf.h"
/*-*****************************
* Decompression functions
*******************************/
FORCE_INLINE size_t LZ5_readStream(int flag, const BYTE** ip, const BYTE* const iend, BYTE* op, BYTE* const oend, const BYTE** streamPtr, const BYTE** streamEnd, int streamFlag)
{
if (!flag) {
if (*ip > iend - 3) return 0;
*streamPtr = *ip + 3;
*streamEnd = *streamPtr + MEM_readLE24(*ip);
if (*streamEnd < *streamPtr) return 0;
*ip = *streamEnd;
#ifdef LZ5_STATS
uncompr_stream[streamFlag] += *streamEnd-*streamPtr;
#else
(void)streamFlag;
#endif
return 1;
} else {
#ifndef LZ5_NO_HUFFMAN
size_t res, streamLen, comprStreamLen;
if (*ip > iend - 6) return 0;
streamLen = MEM_readLE24(*ip);
comprStreamLen = MEM_readLE24(*ip + 3);
// printf("LZ5_readStream ip=%p iout=%p iend=%p streamLen=%d comprStreamLen=%d\n", *ip, *ip + 6 + comprStreamLen, iend, (int)streamLen, (int)comprStreamLen);
if ((op > oend - streamLen) || (*ip + comprStreamLen > iend - 6)) return 0;
res = HUF_decompress(op, streamLen, *ip + 6, comprStreamLen);
if (HUF_isError(res) || (res != streamLen)) return 0;
*ip += comprStreamLen + 6;
*streamPtr = op;
*streamEnd = *streamPtr + streamLen;
#ifdef LZ5_STATS
compr_stream[streamFlag] += comprStreamLen + 6;
decompr_stream[streamFlag] += *streamEnd-*streamPtr;
#endif
return 1;
#else
fprintf(stderr, "compiled with LZ5_NO_HUFFMAN\n");
(void)op; (void)oend;
return 0;
#endif
}
}
FORCE_INLINE int LZ5_decompress_generic(
const char* source,
char* const dest,
int inputSize,
int outputSize, /* this value is the max size of Output Buffer. */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize /* note : = 0 if noDict */
)
{
/* Local Variables */
const BYTE* ip = (const BYTE*) source, *istart = (const BYTE*) source;
const BYTE* const iend = ip + inputSize;
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + outputSize;
BYTE* oexit = op + targetOutputSize;
LZ5_parameters params;
LZ5_dstream_t ctx;
BYTE* decompFlagsBase, *decompOff24Base, *decompOff16Base, *decompLiteralsBase = NULL;
int res, compressionLevel;
if (inputSize < 1) { LZ5_LOG_DECOMPRESS("inputSize=%d outputSize=%d targetOutputSize=%d partialDecoding=%d\n", inputSize, outputSize, targetOutputSize, partialDecoding); return 0; }
compressionLevel = *ip++;
if (compressionLevel < LZ5_MIN_CLEVEL || compressionLevel > LZ5_MAX_CLEVEL) {
LZ5_LOG_DECOMPRESS("ERROR LZ5_decompress_generic inputSize=%d compressionLevel=%d\n", inputSize, compressionLevel);
return -1;
}
LZ5_LOG_DECOMPRESS("LZ5_decompress_generic ip=%p inputSize=%d targetOutputSize=%d dest=%p outputSize=%d cLevel=%d dict=%d dictSize=%d dictStart=%p partialDecoding=%d\n", ip, inputSize, targetOutputSize, dest, outputSize, compressionLevel, dict, (int)dictSize, dictStart, partialDecoding);
decompLiteralsBase = (BYTE*)malloc(4*LZ5_HUF_BLOCK_SIZE);
if (!decompLiteralsBase) return -1;
decompFlagsBase = decompLiteralsBase + LZ5_HUF_BLOCK_SIZE;
decompOff24Base = decompFlagsBase + LZ5_HUF_BLOCK_SIZE;
decompOff16Base = decompOff24Base + LZ5_HUF_BLOCK_SIZE;
#ifdef LZ5_STATS
init_stats();
#endif
(void)istart;
while (ip < iend)
{
res = *ip++;
if (res == LZ5_FLAG_UNCOMPRESSED) /* uncompressed */
{
uint32_t length;
if (ip > iend - 3) { LZ5_LOG_DECOMPRESS("UNCOMPRESSED ip[%p] > iend[%p] - 3\n", ip, iend); goto _output_error; }
length = MEM_readLE24(ip);
ip += 3;
// printf("%d: total=%d block=%d UNCOMPRESSED op=%p oexit=%p oend=%p\n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), length, op, oexit, oend);
if (ip + length > iend || op + length > oend) { LZ5_LOG_DECOMPRESS("UNCOMPRESSED ip[%p]+length[%d] > iend[%p]\n", ip, length, iend); goto _output_error; }
memcpy(op, ip, length);
op += length;
ip += length;
if ((partialDecoding) && (op >= oexit)) break;
#ifdef LZ5_STATS
uncompr_stream[LZ5_STREAM_UNCOMPRESSED] += length;
#endif
continue;
}
if (res&LZ5_FLAG_LEN) {
LZ5_LOG_DECOMPRESS("res=%d\n", res); goto _output_error;
}
if (ip > iend - 5*3) goto _output_error;
ctx.lenPtr = (const BYTE*)ip + 3;
ctx.lenEnd = ctx.lenPtr + MEM_readLE24(ip);
if (ctx.lenEnd < ctx.lenPtr || (ctx.lenEnd > iend - 3)) goto _output_error;
#ifdef LZ5_STATS
uncompr_stream[LZ5_STREAM_LEN] += ctx.lenEnd-ctx.lenPtr + 3;
#endif
ip = ctx.lenEnd;
{ size_t streamLen;
#ifdef LZ5_USE_LOGS
const BYTE* ipos;
size_t comprFlagsLen, comprLiteralsLen, total;
#endif
streamLen = LZ5_readStream(res&LZ5_FLAG_OFFSET16, &ip, iend, decompOff16Base, decompOff16Base + LZ5_HUF_BLOCK_SIZE, &ctx.offset16Ptr, &ctx.offset16End, LZ5_STREAM_OFFSET16);
if (streamLen == 0) goto _output_error;
streamLen = LZ5_readStream(res&LZ5_FLAG_OFFSET24, &ip, iend, decompOff24Base, decompOff24Base + LZ5_HUF_BLOCK_SIZE, &ctx.offset24Ptr, &ctx.offset24End, LZ5_STREAM_OFFSET24);
if (streamLen == 0) goto _output_error;
#ifdef LZ5_USE_LOGS
ipos = ip;
streamLen = LZ5_readStream(res&LZ5_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LZ5_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LZ5_STREAM_FLAGS);
if (streamLen == 0) goto _output_error;
streamLen = (size_t)(ctx.flagsEnd-ctx.flagsPtr);
comprFlagsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos);
ipos = ip;
#else
streamLen = LZ5_readStream(res&LZ5_FLAG_FLAGS, &ip, iend, decompFlagsBase, decompFlagsBase + LZ5_HUF_BLOCK_SIZE, &ctx.flagsPtr, &ctx.flagsEnd, LZ5_STREAM_FLAGS);
if (streamLen == 0) goto _output_error;
#endif
streamLen = LZ5_readStream(res&LZ5_FLAG_LITERALS, &ip, iend, decompLiteralsBase, decompLiteralsBase + LZ5_HUF_BLOCK_SIZE, &ctx.literalsPtr, &ctx.literalsEnd, LZ5_STREAM_LITERALS);
if (streamLen == 0) goto _output_error;
#ifdef LZ5_USE_LOGS
streamLen = (size_t)(ctx.literalsEnd-ctx.literalsPtr);
comprLiteralsLen = ((size_t)(ip - ipos) + 3 >= streamLen) ? 0 : (size_t)(ip - ipos);
total = (size_t)(ip-(ctx.lenEnd-1));
#endif
if (ip > iend) goto _output_error;
LZ5_LOG_DECOMPRESS("%d: total=%d block=%d flagsLen=%d(HUF=%d) literalsLen=%d(HUF=%d) offset16Len=%d offset24Len=%d lengthsLen=%d \n", (int)(op-(BYTE*)dest) ,(int)(ip-istart), (int)total,
(int)(ctx.flagsEnd-ctx.flagsPtr), (int)comprFlagsLen, (int)(ctx.literalsEnd-ctx.literalsPtr), (int)comprLiteralsLen,
(int)(ctx.offset16End-ctx.offset16Ptr), (int)(ctx.offset24End-ctx.offset24Ptr), (int)(ctx.lenEnd-ctx.lenPtr));
}
ctx.last_off = -LZ5_INIT_LAST_OFFSET;
params = LZ5_defaultParameters[compressionLevel - LZ5_MIN_CLEVEL];
if (params.decompressType == LZ5_coderwords_LZ4)
res = LZ5_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel);
else
#ifdef USE_LZ4_ONLY
res = LZ5_decompress_LZ4(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel);
#else
res = LZ5_decompress_LZ5v2(&ctx, op, outputSize, partialDecoding, targetOutputSize, dict, lowPrefix, dictStart, dictSize, compressionLevel);
#endif
LZ5_LOG_DECOMPRESS("LZ5_decompress_generic res=%d inputSize=%d\n", res, (int)(ctx.literalsEnd-ctx.lenEnd));
if (res <= 0) { free(decompLiteralsBase); return res; }
op += res;
outputSize -= res;
if ((partialDecoding) && (op >= oexit)) break;
}
#ifdef LZ5_STATS
print_stats();
#endif
LZ5_LOG_DECOMPRESS("LZ5_decompress_generic total=%d\n", (int)(op-(BYTE*)dest));
free(decompLiteralsBase);
return (int)(op-(BYTE*)dest);
_output_error:
LZ5_LOG_DECOMPRESS("LZ5_decompress_generic ERROR\n");
free(decompLiteralsBase);
return -1;
}
int LZ5_decompress_safe(const char* source, char* dest, int compressedSize, int maxDecompressedSize)
{
return LZ5_decompress_generic(source, dest, compressedSize, maxDecompressedSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
}
int LZ5_decompress_safe_partial(const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize)
{
return LZ5_decompress_generic(source, dest, compressedSize, maxDecompressedSize, partial, targetOutputSize, noDict, (BYTE*)dest, NULL, 0);
}
/*===== streaming decompression functions =====*/
/*
* If you prefer dynamic allocation methods,
* LZ5_createStreamDecode()
* provides a pointer (void*) towards an initialized LZ5_streamDecode_t structure.
*/
LZ5_streamDecode_t* LZ5_createStreamDecode(void)
{
LZ5_streamDecode_t* lz5s = (LZ5_streamDecode_t*) ALLOCATOR(1, sizeof(LZ5_streamDecode_t));
(void)LZ5_count; /* unused function 'LZ5_count' */
return lz5s;
}
int LZ5_freeStreamDecode (LZ5_streamDecode_t* LZ5_stream)
{
FREEMEM(LZ5_stream);
return 0;
}
/*!
* LZ5_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
* This function is not necessary if previous data is still available where it was decoded.
* Loading a size of 0 is allowed (same effect as no dictionary).
* Return : 1 if OK, 0 if error
*/
int LZ5_setStreamDecode (LZ5_streamDecode_t* LZ5_streamDecode, const char* dictionary, int dictSize)
{
LZ5_streamDecode_t* lz5sd = (LZ5_streamDecode_t*) LZ5_streamDecode;
lz5sd->prefixSize = (size_t) dictSize;
lz5sd->prefixEnd = (const BYTE*) dictionary + dictSize;
lz5sd->externalDict = NULL;
lz5sd->extDictSize = 0;
return 1;
}
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks must still be available at the memory position where they were decoded.
If it's not possible, save the relevant part of decoded data into a safe buffer,
and indicate where it stands using LZ5_setStreamDecode()
*/
int LZ5_decompress_safe_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize)
{
LZ5_streamDecode_t* lz5sd = (LZ5_streamDecode_t*) LZ5_streamDecode;
int result;
if (lz5sd->prefixEnd == (BYTE*)dest) {
result = LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize,
full, 0, usingExtDict, lz5sd->prefixEnd - lz5sd->prefixSize, lz5sd->externalDict, lz5sd->extDictSize);
if (result <= 0) return result;
lz5sd->prefixSize += result;
lz5sd->prefixEnd += result;
} else {
lz5sd->extDictSize = lz5sd->prefixSize;
lz5sd->externalDict = lz5sd->prefixEnd - lz5sd->extDictSize;
result = LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize,
full, 0, usingExtDict, (BYTE*)dest, lz5sd->externalDict, lz5sd->extDictSize);
if (result <= 0) return result;
lz5sd->prefixSize = result;
lz5sd->prefixEnd = (BYTE*)dest + result;
}
return result;
}
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as "_continue" ones,
the dictionary must be explicitly provided within parameters
*/
int LZ5_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
if (dictSize==0)
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest, NULL, 0);
if (dictStart+dictSize == dest)
{
if (dictSize >= (int)(LZ5_DICT_SIZE - 1))
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, withPrefix64k, (BYTE*)dest-LZ5_DICT_SIZE, NULL, 0);
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0);
}
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
/* debug function */
int LZ5_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize)
{
return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize);
}
C/lizard/lz5_decompress.h
+159
View File
@@ -0,0 +1,159 @@
/*
LZ5 - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2016, Yann Collet.
Copyright (C) 2016, Przemyslaw Skibinski <inikep@gmail.com>
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
#ifndef LZ5_DECOMPRESS_H_2983
#define LZ5_DECOMPRESS_H_2983
#if defined (__cplusplus)
extern "C" {
#endif
#include "entropy/mem.h" /* U32 */
/*^***************************************************************
* Export parameters
*****************************************************************/
/*
* LZ5_DLL_EXPORT :
* Enable exporting of functions when building a Windows DLL
*/
#if defined(LZ5_DLL_EXPORT) && (LZ5_DLL_EXPORT==1)
# define LZ5DLIB_API __declspec(dllexport)
#elif defined(LZ5_DLL_IMPORT) && (LZ5_DLL_IMPORT==1)
# define LZ5DLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define LZ5DLIB_API
#endif
/*-************************************
* Simple Functions
**************************************/
/*
LZ5_decompress_safe() :
compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
*/
LZ5DLIB_API int LZ5_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*!
LZ5_decompress_safe_partial() :
This function decompress a compressed block of size 'compressedSize' at position 'source'
into destination buffer 'dest' of size 'maxDecompressedSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
*/
LZ5DLIB_API int LZ5_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
/*-**********************************************
* Streaming Decompression Functions
************************************************/
typedef struct {
const BYTE* externalDict;
size_t extDictSize;
const BYTE* prefixEnd;
size_t prefixSize;
} LZ5_streamDecode_t;
/*
* LZ5_streamDecode_t
* information structure to track an LZ5 stream.
* init this structure content using LZ5_setStreamDecode or memset() before first use !
*
* In the context of a DLL (liblz5) please prefer usage of construction methods below.
* They are more future proof, in case of a change of LZ5_streamDecode_t size in the future.
* LZ5_createStreamDecode will allocate and initialize an LZ5_streamDecode_t structure
* LZ5_freeStreamDecode releases its memory.
*/
LZ5DLIB_API LZ5_streamDecode_t* LZ5_createStreamDecode(void);
LZ5DLIB_API int LZ5_freeStreamDecode (LZ5_streamDecode_t* LZ5_stream);
/*! LZ5_setStreamDecode() :
* Use this function to instruct where to find the dictionary.
* Setting a size of 0 is allowed (same effect as reset).
* @return : 1 if OK, 0 if error
*/
LZ5DLIB_API int LZ5_setStreamDecode (LZ5_streamDecode_t* LZ5_streamDecode, const char* dictionary, int dictSize);
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to LZ5_DICT_SIZE)
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including small ones ( < LZ5_DICT_SIZE).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < LZ5_DICT_SIZE).
- _At least_ LZ5_DICT_SIZE + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last LZ5_DICT_SIZE of decoded data into a safe buffer,
and indicate where it is saved using LZ5_setStreamDecode()
*/
LZ5DLIB_API int LZ5_decompress_safe_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
a combination of LZ5_setStreamDecode() followed by LZ5_decompress_x_continue()
They are stand-alone. They don't need nor update an LZ5_streamDecode_t structure.
*/
LZ5DLIB_API int LZ5_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
#if defined (__cplusplus)
}
#endif
#endif /* LZ5_DECOMPRESS_H_2983827168210 */
C/lizard/lz5_decompress_lz4.h
+164
View File
@@ -0,0 +1,164 @@
/*! LZ5_decompress_LZ4() :
* This generic decompression function cover all use cases.
* It shall be instantiated several times, using different sets of directives
* Note that it is important this generic function is really inlined,
* in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int LZ5_decompress_LZ4(
LZ5_dstream_t* ctx,
BYTE* const dest,
int outputSize, /* this value is the max size of Output Buffer. */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize, /* note : = 0 if noDict */
int compressionLevel
)
{
/* Local Variables */
int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr);
const BYTE* const blockBase = ctx->flagsPtr;
const BYTE* const iend = ctx->literalsEnd;
BYTE* op = dest;
BYTE* const oend = op + outputSize;
BYTE* cpy = NULL;
BYTE* oexit = op + targetOutputSize;
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const int checkOffset = (dictSize < (int)(LZ5_DICT_SIZE));
intptr_t length = 0;
(void)compressionLevel;
(void)LZ5_wildCopy;
/* Special cases */
if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */
/* Main Loop : decode sequences */
while (ctx->flagsPtr < ctx->flagsEnd) {
unsigned token;
const BYTE* match;
size_t offset;
/* get literal length */
token = *ctx->flagsPtr++;
if ((length=(token & RUN_MASK_LZ4)) == RUN_MASK_LZ4) {
if (unlikely(ctx->literalsPtr > iend - 5)) { LZ5_LOG_DECOMPRESS_LZ4("0"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += RUN_MASK_LZ4;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LZ5_LOG_DECOMPRESS_LZ4("1"); goto _output_error; } /* overflow detection */
if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LZ5_LOG_DECOMPRESS_LZ4("2"); goto _output_error; } /* overflow detection */
}
/* copy literals */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr + length > iend - (2 + WILDCOPYLENGTH))) { LZ5_LOG_DECOMPRESS_LZ4("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */
#if 1
LZ5_wildCopy16(op, ctx->literalsPtr, cpy);
op = cpy;
ctx->literalsPtr += length;
#else
LZ5_copy8(op, ctx->literalsPtr);
LZ5_copy8(op+8, ctx->literalsPtr+8);
if (length > 16)
LZ5_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy);
op = cpy;
ctx->literalsPtr += length;
#endif
if ((partialDecoding) && (op >= oexit)) return (int) (op-dest);
/* get offset */
offset = MEM_readLE16(ctx->literalsPtr);
ctx->literalsPtr += 2;
match = op - offset;
if ((checkOffset) && (unlikely(match < lowLimit))) { LZ5_LOG_DECOMPRESS_LZ4("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-offset[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)offset); goto _output_error; } /* Error : offset outside buffers */
/* get matchlength */
length = token >> RUN_BITS_LZ4;
if (length == ML_MASK_LZ4) {
if (unlikely(ctx->literalsPtr > iend - 5)) { LZ5_LOG_DECOMPRESS_LZ4("4"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += ML_MASK_LZ4;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LZ5_LOG_DECOMPRESS_LZ4("5"); goto _output_error; } /* overflow detection */
}
length += MINMATCH;
/* check external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LZ5_LOG_DECOMPRESS_LZ4("6"); goto _output_error; } /* doesn't respect parsing restriction */
if (length <= (intptr_t)(lowPrefix - match)) {
/* match can be copied as a single segment from external dictionary */
memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match encompass external dictionary and current block */
size_t const copySize = (size_t)(lowPrefix-match);
size_t const restSize = length - copySize;
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */
BYTE* const endOfMatch = op + restSize;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
} else {
memcpy(op, lowPrefix, restSize);
op += restSize;
} }
continue;
}
/* copy match within block */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LZ5_LOG_DECOMPRESS_LZ4("1match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */
LZ5_copy8(op, match);
LZ5_copy8(op+8, match+8);
if (length > 16)
LZ5_wildCopy16(op + 16, match + 16, cpy);
op = cpy;
if ((partialDecoding) && (op >= oexit)) return (int) (op-dest);
}
/* last literals */
length = ctx->literalsEnd - ctx->literalsPtr;
cpy = op + length;
if ((ctx->literalsPtr+length != iend) || (cpy > oend)) { LZ5_LOG_DECOMPRESS_LZ4("9"); goto _output_error; } /* Error : input must be consumed */
memcpy(op, ctx->literalsPtr, length);
ctx->literalsPtr += length;
op += length;
/* end of decoding */
return (int) (op-dest); /* Nb of output bytes decoded */
/* Overflow error detected */
_output_error:
LZ5_LOG_DECOMPRESS_LZ4("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase);
LZ5_LOG_DECOMPRESS_LZ4("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend);
return (int) (-(ctx->flagsPtr-blockBase))-1;
}
C/lizard/lz5_decompress_lz5v2.h
+224
View File
@@ -0,0 +1,224 @@
/*
[0_MMMM_LLL] - 16-bit offset, 4-bit match length (4-15+), 3-bit literal length (0-7+)
[1_MMMM_LLL] - last offset, 4-bit match length (0-15+), 3-bit literal length (0-7+)
flag 31 - 24-bit offset, match length (47+), no literal length
flag 0-30 - 24-bit offset, 31 match lengths (16-46), no literal length
*/
/*! LZ5_decompress_LZ5v2() :
* This generic decompression function cover all use cases.
* It shall be instantiated several times, using different sets of directives
* Note that it is important this generic function is really inlined,
* in order to remove useless branches during compilation optimization.
*/
FORCE_INLINE int LZ5_decompress_LZ5v2(
LZ5_dstream_t* ctx,
BYTE* const dest,
int outputSize, /* this value is the max size of Output Buffer. */
int partialDecoding, /* full, partial */
int targetOutputSize, /* only used if partialDecoding==partial */
int dict, /* noDict, withPrefix64k, usingExtDict */
const BYTE* const lowPrefix, /* == dest if dict == noDict */
const BYTE* const dictStart, /* only if dict==usingExtDict */
const size_t dictSize, /* note : = 0 if noDict */
int compressionLevel
)
{
/* Local Variables */
int inputSize = (int)(ctx->flagsEnd - ctx->flagsPtr);
const BYTE* const blockBase = ctx->flagsPtr;
const BYTE* const iend = ctx->literalsEnd;
BYTE* op = dest;
BYTE* const oend = op + outputSize;
BYTE* cpy = NULL;
BYTE* oexit = op + targetOutputSize;
const BYTE* const lowLimit = lowPrefix - dictSize;
const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize;
const int checkOffset = (dictSize < (int)(LZ5_DICT_SIZE));
intptr_t last_off = ctx->last_off;
intptr_t length = 0;
(void)compressionLevel;
(void)LZ5_wildCopy;
/* Special cases */
if (unlikely(outputSize==0)) return ((inputSize==1) && (*ctx->flagsPtr==0)) ? 0 : -1; /* Empty output buffer */
/* Main Loop : decode sequences */
while (ctx->flagsPtr < ctx->flagsEnd) {
unsigned token;
const BYTE* match;
// intptr_t litLength;
if ((partialDecoding) && (op >= oexit)) return (int) (op-dest);
/* get literal length */
token = *ctx->flagsPtr++;
// LZ5_LOG_DECOMPRESS_LZ5v2("token : %u\n", (U32)token);
if (token >= 32)
{
if ((length=(token & MAX_SHORT_LITLEN)) == MAX_SHORT_LITLEN) {
if (unlikely(ctx->literalsPtr > iend - 1)) { LZ5_LOG_DECOMPRESS_LZ5v2("1"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += MAX_SHORT_LITLEN;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LZ5_LOG_DECOMPRESS_LZ5v2("2"); goto _output_error; } /* overflow detection */
if (unlikely((size_t)(ctx->literalsPtr+length)<(size_t)(ctx->literalsPtr))) { LZ5_LOG_DECOMPRESS_LZ5v2("3"); goto _output_error; } /* overflow detection */
}
/* copy literals */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH || ctx->literalsPtr > iend - WILDCOPYLENGTH)) { LZ5_LOG_DECOMPRESS_LZ5v2("offset outside buffers\n"); goto _output_error; } /* Error : offset outside buffers */
#if 1
LZ5_wildCopy16(op, ctx->literalsPtr, cpy);
op = cpy;
ctx->literalsPtr += length;
#else
LZ5_copy8(op, ctx->literalsPtr);
LZ5_copy8(op+8, ctx->literalsPtr+8);
if (length > 16)
LZ5_wildCopy16(op + 16, ctx->literalsPtr + 16, cpy);
op = cpy;
ctx->literalsPtr += length;
#endif
/* get offset */
if (unlikely(ctx->offset16Ptr > ctx->offset16End)) { LZ5_LOG_DECOMPRESS_LZ5v2("(ctx->offset16Ptr > ctx->offset16End\n"); goto _output_error; }
#if 1
{ /* branchless */
intptr_t new_off = MEM_readLE16(ctx->offset16Ptr);
uintptr_t not_repCode = (uintptr_t)(token >> ML_RUN_BITS) - 1;
last_off ^= not_repCode & (last_off ^ -new_off);
ctx->offset16Ptr = (BYTE*)((uintptr_t)ctx->offset16Ptr + (not_repCode & 2));
}
#else
if ((token >> ML_RUN_BITS_LZ5v2) == 0)
{
last_off = -(intptr_t)MEM_readLE16(ctx->offset16Ptr);
ctx->offset16Ptr += 2;
// LZ5v2_DEBUG("MEM_readLE16 offset=%d\n", (int)offset);
}
#endif
/* get matchlength */
length = (token >> RUN_BITS_LZ5v2) & MAX_SHORT_MATCHLEN;
// printf("length=%d token=%d\n", (int)length, (int)token);
if (length == MAX_SHORT_MATCHLEN) {
if (unlikely(ctx->literalsPtr > iend - 1)) { LZ5_LOG_DECOMPRESS_LZ5v2("6"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
length += MAX_SHORT_MATCHLEN;
ctx->literalsPtr++;
if (unlikely((size_t)(op+length)<(size_t)(op))) { LZ5_LOG_DECOMPRESS_LZ5v2("7"); goto _output_error; } /* overflow detection */
}
DECOMPLOG_CODEWORDS_LZ5v2("T32+ literal=%u match=%u offset=%d ipos=%d opos=%d\n", (U32)litLength, (U32)length, (int)-last_off, (U32)(ctx->flagsPtr-blockBase), (U32)(op-dest));
}
else
if (token < LZ5_LAST_LONG_OFF)
{
if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LZ5_LOG_DECOMPRESS_LZ5v2("8"); goto _output_error; }
length = token + MM_LONGOFF;
last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr);
ctx->offset24Ptr += 3;
DECOMPLOG_CODEWORDS_LZ5v2("T0-30 literal=%u match=%u offset=%d\n", 0, (U32)length, (int)-last_off);
}
else
{
if (unlikely(ctx->literalsPtr > iend - 1)) { LZ5_LOG_DECOMPRESS_LZ5v2("9"); goto _output_error; }
length = *ctx->literalsPtr;
if unlikely(length >= 254) {
if (length == 254) {
length = MEM_readLE16(ctx->literalsPtr+1);
ctx->literalsPtr += 2;
} else {
length = MEM_readLE24(ctx->literalsPtr+1);
ctx->literalsPtr += 3;
}
}
ctx->literalsPtr++;
length += LZ5_LAST_LONG_OFF + MM_LONGOFF;
if (unlikely(ctx->offset24Ptr > ctx->offset24End - 3)) { LZ5_LOG_DECOMPRESS_LZ5v2("10"); goto _output_error; }
last_off = -(intptr_t)MEM_readLE24(ctx->offset24Ptr);
ctx->offset24Ptr += 3;
}
match = op + last_off;
if ((checkOffset) && ((unlikely((uintptr_t)(-last_off) > (uintptr_t)op) || (match < lowLimit)))) { LZ5_LOG_DECOMPRESS_LZ5v2("lowPrefix[%p]-dictSize[%d]=lowLimit[%p] match[%p]=op[%p]-last_off[%d]\n", lowPrefix, (int)dictSize, lowLimit, match, op, (int)last_off); goto _output_error; } /* Error : offset outside buffers */
/* check external dictionary */
if ((dict==usingExtDict) && (match < lowPrefix)) {
if (unlikely(op + length > oend - WILDCOPYLENGTH)) { LZ5_LOG_DECOMPRESS_LZ5v2("12"); goto _output_error; } /* doesn't respect parsing restriction */
if (length <= (intptr_t)(lowPrefix - match)) {
/* match can be copied as a single segment from external dictionary */
memmove(op, dictEnd - (lowPrefix-match), length);
op += length;
} else {
/* match encompass external dictionary and current block */
size_t const copySize = (size_t)(lowPrefix-match);
size_t const restSize = length - copySize;
memcpy(op, dictEnd - copySize, copySize);
op += copySize;
if (restSize > (size_t)(op-lowPrefix)) { /* overlap copy */
BYTE* const endOfMatch = op + restSize;
const BYTE* copyFrom = lowPrefix;
while (op < endOfMatch) *op++ = *copyFrom++;
} else {
memcpy(op, lowPrefix, restSize);
op += restSize;
} }
continue;
}
/* copy match within block */
cpy = op + length;
if (unlikely(cpy > oend - WILDCOPYLENGTH)) { LZ5_LOG_DECOMPRESS_LZ5v2("13match=%p lowLimit=%p\n", match, lowLimit); goto _output_error; } /* Error : offset outside buffers */
LZ5_copy8(op, match);
LZ5_copy8(op+8, match+8);
if (length > 16)
LZ5_wildCopy16(op + 16, match + 16, cpy);
op = cpy;
}
/* last literals */
length = ctx->literalsEnd - ctx->literalsPtr;
cpy = op + length;
if ((ctx->literalsPtr+length != iend) || (cpy > oend)) { LZ5_LOG_DECOMPRESS_LZ5v2("14"); goto _output_error; } /* Error : input must be consumed */
memcpy(op, ctx->literalsPtr, length);
ctx->literalsPtr += length;
op += length;
/* end of decoding */
ctx->last_off = last_off;
return (int) (op-dest); /* Nb of output bytes decoded */
/* Overflow error detected */
_output_error:
LZ5_LOG_DECOMPRESS_LZ5v2("_output_error=%d ctx->flagsPtr=%p blockBase=%p\n", (int) (-(ctx->flagsPtr-blockBase))-1, ctx->flagsPtr, blockBase);
LZ5_LOG_DECOMPRESS_LZ5v2("cpy=%p oend=%p ctx->literalsPtr+length[%d]=%p iend=%p\n", cpy, oend, (int)length, ctx->literalsPtr+length, iend);
return (int) (-(ctx->flagsPtr-blockBase))-1;
}
C/lizard/lz5_parser_fast.h
+200
View File
@@ -0,0 +1,200 @@
#define LZ5_FAST_MIN_OFFSET 8
#define LZ5_FAST_LONGOFF_MM 0 /* not used with offsets > 1<<16 */
/**************************************
* Hash Functions
**************************************/
static size_t LZ5_hashPosition(const void* p)
{
if (MEM_64bits())
return LZ5_hash5Ptr(p, LZ5_HASHLOG_LZ4);
return LZ5_hash4Ptr(p, LZ5_HASHLOG_LZ4);
}
static void LZ5_putPositionOnHash(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase)
{
hashTable[h] = (U32)(p-srcBase);
}
static void LZ5_putPosition(const BYTE* p, U32* hashTable, const BYTE* srcBase)
{
size_t const h = LZ5_hashPosition(p);
LZ5_putPositionOnHash(p, h, hashTable, srcBase);
}
static U32 LZ5_getPositionOnHash(size_t h, U32* hashTable)
{
return hashTable[h];
}
static U32 LZ5_getPosition(const BYTE* p, U32* hashTable)
{
size_t const h = LZ5_hashPosition(p);
return LZ5_getPositionOnHash(h, hashTable);
}
static const U32 LZ5_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */
static const U32 LZ5_minLength = (MFLIMIT+1);
FORCE_INLINE int LZ5_compress_fast(
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const U32 acceleration = 1;
const BYTE* base = ctx->base;
const U32 lowLimit = ctx->lowLimit;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
const BYTE* anchor = ip;
size_t forwardH, matchIndex;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
// fprintf(stderr, "base=%p LZ5_stream_t=%d inputSize=%d maxOutputSize=%d\n", base, sizeof(LZ5_stream_t), inputSize, maxOutputSize);
// fprintf(stderr, "ip=%d base=%p lowPrefixPtr=%p dictBase=%d lowLimit=%p op=%p\n", ip, base, lowPrefixPtr, lowLimit, dictBase, op);
/* Init conditions */
if ((U32)(iend-ip) > (U32)LZ5_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */
if ((U32)(iend-ip) < LZ5_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
LZ5_putPosition(ip, ctx->hashTable, base);
ip++; forwardH = LZ5_hashPosition(ip);
/* Main Loop */
for ( ; ; ) {
const BYTE* match;
// BYTE* token;
size_t matchLength;
/* Find a match */
{ const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ5_skipTrigger;
while (1) {
size_t const h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> LZ5_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
matchIndex = LZ5_getPositionOnHash(h, ctx->hashTable);
forwardH = LZ5_hashPosition(forwardIp);
LZ5_putPositionOnHash(ip, h, ctx->hashTable, base);
if ((matchIndex < lowLimit) || (base + matchIndex + maxDistance < ip)) continue;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_FAST_MIN_OFFSET)
#endif
if (MEM_read32(match) == MEM_read32(ip))
{
int back = 0;
matchLength = LZ5_count(ip+MINMATCH, match+MINMATCH, matchlimit);
while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
{
ip += back;
match += back;
break;
}
}
} else {
match = dictBase + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_FAST_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance;
int back = 0;
matchLength = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
match = base + matchIndex + back;
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
{
ip += back;
break;
}
}
}
} // while (1)
}
_next_match:
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error;
/* Test end of chunk */
if (ip > mflimit) break;
/* Fill table */
LZ5_putPosition(ip-2, ctx->hashTable, base);
/* Test next position */
matchIndex = LZ5_getPosition(ip, ctx->hashTable);
LZ5_putPosition(ip, ctx->hashTable, base);
if (matchIndex >= lowLimit && (base + matchIndex + maxDistance >= ip))
{
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_FAST_MIN_OFFSET)
#endif
if (MEM_read32(match) == MEM_read32(ip))
{
matchLength = LZ5_count(ip+MINMATCH, match+MINMATCH, matchlimit);
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
goto _next_match;
}
} else {
match = dictBase + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_FAST_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
matchLength = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
match = base + matchIndex;
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
goto _next_match;
}
}
}
/* Prepare next loop */
forwardH = LZ5_hashPosition(++ip);
}
_last_literals:
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_fastbig.h
+179
View File
@@ -0,0 +1,179 @@
#define LZ5_FASTBIG_LONGOFF_MM MM_LONGOFF
/**************************************
* Hash Functions
**************************************/
static size_t LZ5_hashPositionHLog(const void* p, int hashLog)
{
if (MEM_64bits())
return LZ5_hash5Ptr(p, hashLog);
return LZ5_hash4Ptr(p, hashLog);
}
static void LZ5_putPositionOnHashHLog(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase)
{
hashTable[h] = (U32)(p-srcBase);
}
static void LZ5_putPositionHLog(const BYTE* p, U32* hashTable, const BYTE* srcBase, int hashLog)
{
size_t const h = LZ5_hashPositionHLog(p, hashLog);
LZ5_putPositionOnHashHLog(p, h, hashTable, srcBase);
}
static U32 LZ5_getPositionOnHashHLog(size_t h, U32* hashTable)
{
return hashTable[h];
}
static U32 LZ5_getPositionHLog(const BYTE* p, U32* hashTable, int hashLog)
{
size_t const h = LZ5_hashPositionHLog(p, hashLog);
return LZ5_getPositionOnHashHLog(h, hashTable);
}
FORCE_INLINE int LZ5_compress_fastBig(
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const U32 acceleration = 1;
const BYTE* base = ctx->base;
const U32 lowLimit = ctx->lowLimit;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
const BYTE* anchor = ip;
size_t forwardH, matchIndex;
const int hashLog = ctx->params.hashLog;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
// fprintf(stderr, "base=%p LZ5_stream_t=%d inputSize=%d maxOutputSize=%d\n", base, sizeof(LZ5_stream_t), inputSize, maxOutputSize);
// fprintf(stderr, "ip=%d base=%p lowPrefixPtr=%p dictBase=%d lowLimit=%p op=%p\n", ip, base, lowPrefixPtr, lowLimit, dictBase, op);
/* Init conditions */
if ((U32)(iend-ip) > (U32)LZ5_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */
if ((U32)(iend-ip) < LZ5_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
LZ5_putPositionHLog(ip, ctx->hashTable, base, hashLog);
ip++; forwardH = LZ5_hashPositionHLog(ip, hashLog);
/* Main Loop */
for ( ; ; ) {
const BYTE* match;
// BYTE* token;
size_t matchLength;
/* Find a match */
{ const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ5_skipTrigger;
while (1) {
size_t const h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> LZ5_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
matchIndex = LZ5_getPositionOnHashHLog(h, ctx->hashTable);
forwardH = LZ5_hashPositionHLog(forwardIp, hashLog);
LZ5_putPositionOnHashHLog(ip, h, ctx->hashTable, base);
if ((matchIndex < lowLimit) || (base + matchIndex + maxDistance < ip)) continue;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_FAST_MIN_OFFSET)
if (MEM_read32(match) == MEM_read32(ip))
{
int back = 0;
matchLength = LZ5_count(ip+MINMATCH, match+MINMATCH, matchlimit);
while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
if ((matchLength >= LZ5_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
{
ip += back;
match += back;
break;
}
}
} else {
match = dictBase + matchIndex;
if ((U32)(ip - (base + matchIndex)) >= LZ5_FAST_MIN_OFFSET)
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance;
int back = 0;
matchLength = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
match = base + matchIndex + back;
if ((matchLength >= LZ5_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
{
ip += back;
break;
}
}
}
} // while (1)
}
_next_match:
if (LZ5_encodeSequence_LZ5v2(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error;
/* Test end of chunk */
if (ip > mflimit) break;
/* Fill table */
LZ5_putPositionHLog(ip-2, ctx->hashTable, base, hashLog);
/* Test next position */
matchIndex = LZ5_getPositionHLog(ip, ctx->hashTable, hashLog);
LZ5_putPositionHLog(ip, ctx->hashTable, base, hashLog);
if (matchIndex >= lowLimit && (base + matchIndex + maxDistance >= ip))
{
if (matchIndex >= dictLimit) {
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_FAST_MIN_OFFSET)
if (MEM_read32(match) == MEM_read32(ip))
{
matchLength = LZ5_count(ip+MINMATCH, match+MINMATCH, matchlimit);
if ((matchLength >= LZ5_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
goto _next_match;
}
} else {
match = dictBase + matchIndex;
if ((U32)(ip - (base + matchIndex)) >= LZ5_FAST_MIN_OFFSET)
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
matchLength = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
match = base + matchIndex;
if ((matchLength >= LZ5_FASTBIG_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
goto _next_match;
}
}
}
/* Prepare next loop */
forwardH = LZ5_hashPositionHLog(++ip, hashLog);
}
_last_literals:
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ5v2(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_fastsmall.h
+193
View File
@@ -0,0 +1,193 @@
/**************************************
* Hash Functions
**************************************/
static size_t LZ5_hashPositionSmall(const void* p)
{
if (MEM_64bits())
return LZ5_hash5Ptr(p, LZ5_HASHLOG_LZ4SM);
return LZ5_hash4Ptr(p, LZ5_HASHLOG_LZ4SM);
}
static void LZ5_putPositionOnHashSmall(const BYTE* p, size_t h, U32* hashTable, const BYTE* srcBase)
{
hashTable[h] = (U32)(p-srcBase);
}
static void LZ5_putPositionSmall(const BYTE* p, U32* hashTable, const BYTE* srcBase)
{
size_t const h = LZ5_hashPositionSmall(p);
LZ5_putPositionOnHashSmall(p, h, hashTable, srcBase);
}
static U32 LZ5_getPositionOnHashSmall(size_t h, U32* hashTable)
{
return hashTable[h];
}
static U32 LZ5_getPositionSmall(const BYTE* p, U32* hashTable)
{
size_t const h = LZ5_hashPositionSmall(p);
return LZ5_getPositionOnHashSmall(h, hashTable);
}
FORCE_INLINE int LZ5_compress_fastSmall(
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const U32 acceleration = 1;
const BYTE* base = ctx->base;
const U32 lowLimit = ctx->lowLimit;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = iend - LASTLITERALS;
const BYTE* anchor = ip;
size_t forwardH, matchIndex;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
// fprintf(stderr, "base=%p LZ5_stream_t=%d inputSize=%d maxOutputSize=%d\n", base, sizeof(LZ5_stream_t), inputSize, maxOutputSize);
// fprintf(stderr, "ip=%d base=%p lowPrefixPtr=%p dictBase=%d lowLimit=%p op=%p\n", ip, base, lowPrefixPtr, lowLimit, dictBase, op);
/* Init conditions */
if ((U32)(iend-ip) > (U32)LZ5_MAX_INPUT_SIZE) goto _output_error; /* Unsupported inputSize, too large (or negative) */
if ((U32)(iend-ip) < LZ5_minLength) goto _last_literals; /* Input too small, no compression (all literals) */
/* First Byte */
LZ5_putPositionSmall(ip, ctx->hashTable, base);
ip++; forwardH = LZ5_hashPositionSmall(ip);
/* Main Loop */
for ( ; ; ) {
const BYTE* match;
// BYTE* token;
size_t matchLength;
/* Find a match */
{ const BYTE* forwardIp = ip;
unsigned step = 1;
unsigned searchMatchNb = acceleration << LZ5_skipTrigger;
while (1) {
size_t const h = forwardH;
ip = forwardIp;
forwardIp += step;
step = (searchMatchNb++ >> LZ5_skipTrigger);
if (unlikely(forwardIp > mflimit)) goto _last_literals;
matchIndex = LZ5_getPositionOnHashSmall(h, ctx->hashTable);
forwardH = LZ5_hashPositionSmall(forwardIp);
LZ5_putPositionOnHashSmall(ip, h, ctx->hashTable, base);
if ((matchIndex < lowLimit) || (base + matchIndex + maxDistance < ip)) continue;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_FAST_MIN_OFFSET)
#endif
if (MEM_read32(match) == MEM_read32(ip))
{
int back = 0;
matchLength = LZ5_count(ip+MINMATCH, match+MINMATCH, matchlimit);
while ((ip+back > anchor) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
{
ip += back;
match += back;
break;
}
}
} else {
match = dictBase + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_FAST_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
const U32 newLowLimit = (lowLimit + maxDistance >= (U32)(ip-base)) ? lowLimit : (U32)(ip - base) - maxDistance;
int back = 0;
matchLength = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
while ((ip+back > anchor) && (matchIndex+back > newLowLimit) && (ip[back-1] == match[back-1])) back--;
matchLength -= back;
match = base + matchIndex + back;
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
{
ip += back;
break;
}
}
}
} // while (1)
}
_next_match:
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, matchLength+MINMATCH, match)) goto _output_error;
/* Test end of chunk */
if (ip > mflimit) break;
/* Fill table */
LZ5_putPositionSmall(ip-2, ctx->hashTable, base);
/* Test next position */
matchIndex = LZ5_getPositionSmall(ip, ctx->hashTable);
LZ5_putPositionSmall(ip, ctx->hashTable, base);
if (matchIndex >= lowLimit && (base + matchIndex + maxDistance >= ip))
{
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_FAST_MIN_OFFSET)
#endif
if (MEM_read32(match) == MEM_read32(ip))
{
matchLength = LZ5_count(ip+MINMATCH, match+MINMATCH, matchlimit);
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
goto _next_match;
}
} else {
match = dictBase + matchIndex;
#if LZ5_FAST_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_FAST_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
matchLength = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, matchlimit, dictEnd, lowPrefixPtr);
match = base + matchIndex;
#if LZ5_FAST_LONGOFF_MM > 0
if ((matchLength >= LZ5_FAST_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
goto _next_match;
}
}
}
/* Prepare next loop */
forwardH = LZ5_hashPositionSmall(++ip);
}
_last_literals:
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_hashchain.h
+368
View File
@@ -0,0 +1,368 @@
#define LZ5_HC_MIN_OFFSET 8
#define LZ5_HC_LONGOFF_MM 0 /* not used with offsets > 1<<16 */
#define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH)
#define GET_MINMATCH(offset) (MINMATCH)
#if 1
#define LZ5_HC_HASH_FUNCTION(ip, hashLog) LZ5_hashPtr(ip, hashLog, ctx->params.searchLength)
#else
#define LZ5_HC_HASH_FUNCTION(ip, hashLog) LZ5_hash5Ptr(ip, hashLog)
#endif
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ5_Insert (LZ5_stream_t* ctx, const BYTE* ip)
{
U32* const chainTable = ctx->chainTable;
U32* const hashTable = ctx->hashTable;
#if MINMATCH == 3
U32* HashTable3 = ctx->hashTable3;
#endif
const BYTE* const base = ctx->base;
U32 const target = (U32)(ip - base);
U32 idx = ctx->nextToUpdate;
const int hashLog = ctx->params.hashLog;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
while (idx < target) {
size_t const h = LZ5_hashPtr(base+idx, hashLog, ctx->params.searchLength);
size_t delta = idx - hashTable[h];
if (delta>maxDistance) delta = maxDistance;
DELTANEXT(idx) = (U32)delta;
if (idx >= hashTable[h] + LZ5_HC_MIN_OFFSET)
hashTable[h] = idx;
#if MINMATCH == 3
HashTable3[LZ5_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx;
#endif
idx++;
}
ctx->nextToUpdate = target;
}
FORCE_INLINE int LZ5_InsertAndFindBestMatch (LZ5_stream_t* ctx, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos)
{
U32* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
U32 matchIndex, delta;
const BYTE* match;
int nbAttempts=ctx->params.searchNum;
size_t ml=0;
const int hashLog = ctx->params.hashLog;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance;
/* HC4 match finder */
LZ5_Insert(ctx, ip);
matchIndex = HashTable[LZ5_HC_HASH_FUNCTION(ip, hashLog)];
while ((matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LZ5_HC_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_HC_MIN_OFFSET)
#endif
if (*(match+ml) == *(ip+ml)
&& (MEM_read32(match) == MEM_read32(ip)))
{
size_t const mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
#if LZ5_HC_LONGOFF_MM > 0
if ((mlt >= LZ5_HC_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
} else {
match = dictBase + matchIndex;
// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match));
#if LZ5_HC_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_HC_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
size_t mlt = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH;
#if LZ5_HC_LONGOFF_MM > 0
if ((mlt >= LZ5_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LZ5_MAX_16BIT_OFFSET))
#endif
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
delta = DELTANEXT(matchIndex);
if (delta > matchIndex) break;
matchIndex -= delta;
}
return (int)ml;
}
FORCE_INLINE int LZ5_InsertAndGetWiderMatch (
LZ5_stream_t* ctx,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
int longest,
const BYTE** matchpos,
const BYTE** startpos)
{
U32* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
U32 matchIndex, delta;
int nbAttempts = ctx->params.searchNum;
int LLdelta = (int)(ip-iLowLimit);
const int hashLog = ctx->params.hashLog;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance;
/* First Match */
LZ5_Insert(ctx, ip);
matchIndex = HashTable[LZ5_HC_HASH_FUNCTION(ip, hashLog)];
while ((matchIndex>=lowLimit) && (nbAttempts)) {
nbAttempts--;
if (matchIndex >= dictLimit) {
const BYTE* match = base + matchIndex;
#if LZ5_HC_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_HC_MIN_OFFSET)
#endif
if (*(iLowLimit + longest) == *(match - LLdelta + longest)) {
if (MEM_read32(match) == MEM_read32(ip)) {
int mlt = MINMATCH + LZ5_count(ip+MINMATCH, match+MINMATCH, iHighLimit);
int back = 0;
while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
#if LZ5_HC_LONGOFF_MM > 0
if ((mlt >= LZ5_HC_LONGOFF_MM) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
#endif
if (mlt > longest) {
longest = (int)mlt;
*matchpos = match+back;
*startpos = ip+back;
}
}
}
} else {
const BYTE* match = dictBase + matchIndex;
#if LZ5_HC_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_HC_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
int back=0;
size_t mlt = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH;
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
#if LZ5_HC_LONGOFF_MM > 0
if ((mlt >= LZ5_HC_LONGOFF_MM) || ((U32)(ip - (base + matchIndex)) < LZ5_MAX_16BIT_OFFSET))
#endif
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
delta = DELTANEXT(matchIndex);
if (delta > matchIndex) break;
matchIndex -= delta;
}
return longest;
}
FORCE_INLINE int LZ5_compress_hashChain (
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const BYTE* anchor = ip;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
int ml, ml2, ml3, ml0;
const BYTE* ref = NULL;
const BYTE* start2 = NULL;
const BYTE* ref2 = NULL;
const BYTE* start3 = NULL;
const BYTE* ref3 = NULL;
const BYTE* start0;
const BYTE* ref0;
/* init */
ip++;
/* Main Loop */
while (ip < mflimit) {
ml = LZ5_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref));
if (!ml) { ip++; continue; }
/* saved, in case we would skip too much */
start0 = ip;
ref0 = ref;
ml0 = ml;
_Search2:
if (ip+ml < mflimit)
ml2 = LZ5_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);
else ml2 = ml;
if (ml2 == ml) { /* No better match */
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
continue;
}
if (start0 < ip) {
if (start2 < ip + ml0) { /* empirical */
ip = start0;
ref = ref0;
ml = ml0;
}
}
/* Here, start0==ip */
if ((start2 - ip) < 3) { /* First Match too small : removed */
ml = ml2;
ip = start2;
ref =ref2;
goto _Search2;
}
_Search3:
/*
* Currently we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
if ((start2 - ip) < OPTIMAL_ML) {
int correction;
int new_ml = ml;
if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
if (ip+new_ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) {
new_ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2));
if (new_ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
continue;
}
}
correction = new_ml - (int)(start2 - ip);
if (correction > 0) {
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
}
/* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
if (start2 + ml2 < mflimit)
ml3 = LZ5_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);
else ml3 = ml2;
if (ml3 == ml2) { /* No better match : 2 sequences to encode */
/* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start2;
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0;
continue;
}
if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */
if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
if (start2 < ip+ml) {
int correction = (int)(ip+ml - start2);
start2 += correction;
ref2 += correction;
ml2 -= correction;
if (ml2 < GET_MINMATCH((U32)(start2 - ref2))) {
start2 = start3;
ref2 = ref3;
ml2 = ml3;
}
}
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start3;
ref = ref3;
ml = ml3;
start0 = start2;
ref0 = ref2;
ml0 = ml2;
goto _Search2;
}
start2 = start3;
ref2 = ref3;
ml2 = ml3;
goto _Search3;
}
/*
* OK, now we have 3 ascending matches; let's write at least the first one
* ip & ref are known; Now for ml
*/
if (start2 < ip+ml) {
if ((start2 - ip) < (int)ML_MASK_LZ4) {
int correction;
if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
if (ip + ml > start2 + ml2 - GET_MINMATCH((U32)(start2 - ref2))) {
ml = (int)(start2 - ip) + ml2 - GET_MINMATCH((U32)(start2 - ref2));
if (ml < GET_MINMATCH((U32)(ip - ref))) { // match2 doesn't fit, remove it
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start3;
ref = ref3;
ml = ml3;
start0 = start2;
ref0 = ref2;
ml0 = ml2;
goto _Search2;
}
}
correction = ml - (int)(start2 - ip);
if (correction > 0) {
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
} else {
ml = (int)(start2 - ip);
}
}
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start2;
ref = ref2;
ml = ml2;
start2 = start3;
ref2 = ref3;
ml2 = ml3;
goto _Search3;
}
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_lowestprice.h
+380
View File
@@ -0,0 +1,380 @@
#define LZ5_LOWESTPRICE_MIN_OFFSET 8
FORCE_INLINE size_t LZ5_more_profitable(LZ5_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, size_t literals, int last_off)
{
size_t sum;
if (literals > 0)
sum = MAX(common + literals, best_common);
else
sum = MAX(common, best_common - literals);
if ((int)off == last_off) off = 0; // rep code
if ((int)best_off == last_off) best_off = 0;
return LZ5_get_price_LZ5v2(ctx, last_off, ip, ctx->off24pos, sum - common, (U32)off, common) <= LZ5_get_price_LZ5v2(ctx, last_off, best_ip, ctx->off24pos, sum - best_common, (U32)best_off, best_common);
}
FORCE_INLINE size_t LZ5_better_price(LZ5_stream_t* const ctx, const BYTE *best_ip, size_t best_off, size_t best_common, const BYTE *ip, size_t off, size_t common, int last_off)
{
if ((int)off == last_off) off = 0; // rep code
if ((int)best_off == last_off) best_off = 0;
return LZ5_get_price_LZ5v2(ctx, last_off, ip, ctx->off24pos, 0, (U32)off, common) < LZ5_get_price_LZ5v2(ctx, last_off, best_ip, ctx->off24pos, common - best_common, (U32)best_off, best_common);
}
FORCE_INLINE int LZ5_FindMatchLowestPrice (LZ5_stream_t* ctx, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos)
{
U32* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const intptr_t dictLimit = ctx->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1;
const intptr_t current = (ip - base);
const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance;
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
intptr_t matchIndex;
const BYTE* match, *matchDict;
int nbAttempts=ctx->params.searchNum;
size_t ml=0, mlt;
matchIndex = HashTable[LZ5_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)];
if (ctx->last_off >= LZ5_LOWESTPRICE_MIN_OFFSET) {
intptr_t matchIndexLO = (ip - ctx->last_off) - base;
if (matchIndexLO >= lowLimit) {
if (matchIndexLO >= dictLimit) {
match = base + matchIndexLO;
mlt = LZ5_count(ip, match, iLimit);// + MINMATCH;
// if ((mlt >= minMatchLongOff) || (ctx->last_off < LZ5_MAX_16BIT_OFFSET))
if (mlt > REPMINMATCH) {
*matchpos = match;
return (int)mlt;
}
} else {
match = dictBase + matchIndexLO;
if ((U32)((dictLimit-1) - matchIndexLO) >= 3) { /* intentional overflow */
mlt = LZ5_count_2segments(ip, match, iLimit, dictEnd, lowPrefixPtr);
// if ((mlt >= minMatchLongOff) || (ctx->last_off < LZ5_MAX_16BIT_OFFSET))
if (mlt > REPMINMATCH) {
*matchpos = base + matchIndexLO; /* virtual matchpos */
return (int)mlt;
}
}
}
}
}
#if MINMATCH == 3
{
U32 matchIndex3 = ctx->hashTable3[LZ5_hash3Ptr(ip, ctx->params.hashLog3)];
if (matchIndex3 < current && matchIndex3 >= lowLimit)
{
size_t offset = (size_t)current - matchIndex3;
if (offset < LZ5_MAX_8BIT_OFFSET)
{
match = ip - offset;
if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match))
{
ml = 3;//LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
*matchpos = match;
}
}
}
}
#endif
while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) {
nbAttempts--;
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_LOWESTPRICE_MIN_OFFSET) {
if (matchIndex >= dictLimit) {
if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) {
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (!ml || (mlt > ml && LZ5_better_price(ctx, ip, (ip - *matchpos), ml, ip, (ip - match), mlt, ctx->last_off)))
{ ml = mlt; *matchpos = match; }
}
} else {
matchDict = dictBase + matchIndex;
// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match));
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(matchDict) == MEM_read32(ip)) {
mlt = LZ5_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (!ml || (mlt > ml && LZ5_better_price(ctx, ip, (ip - *matchpos), ml, ip, (U32)(ip - match), mlt, ctx->last_off)))
{ ml = mlt; *matchpos = match; } /* virtual matchpos */
}
}
}
matchIndex -= chainTable[matchIndex & contentMask];
}
return (int)ml;
}
FORCE_INLINE size_t LZ5_GetWiderMatch (
LZ5_stream_t* ctx,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
size_t longest,
const BYTE** matchpos,
const BYTE** startpos)
{
U32* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const intptr_t dictLimit = ctx->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1;
const intptr_t current = (ip - base);
const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance;
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const BYTE* match, *matchDict;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
intptr_t matchIndex;
int nbAttempts = ctx->params.searchNum;
size_t mlt;
/* First Match */
matchIndex = HashTable[LZ5_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)];
if (ctx->last_off >= LZ5_LOWESTPRICE_MIN_OFFSET) {
intptr_t matchIndexLO = (ip - ctx->last_off) - base;
if (matchIndexLO >= lowLimit) {
if (matchIndexLO >= dictLimit) {
match = base + matchIndexLO;
if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) {
int back = 0;
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH;
while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if (mlt > longest)
if ((mlt >= minMatchLongOff) || (ctx->last_off < LZ5_MAX_16BIT_OFFSET)) {
*matchpos = match+back;
*startpos = ip+back;
longest = mlt;
}
}
} else {
match = dictBase + matchIndexLO;
if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */
if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) {
int back=0;
mlt = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH;
while ((ip+back > iLowLimit) && (matchIndexLO+back > lowLimit) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if (mlt > longest)
if ((mlt >= minMatchLongOff) || (ctx->last_off < LZ5_MAX_16BIT_OFFSET)) {
*matchpos = base + matchIndexLO + back; /* virtual matchpos */
*startpos = ip+back;
longest = mlt;
}
}
}
}
}
#if MINMATCH == 3
{
U32 matchIndex3 = ctx->hashTable3[LZ5_hash3Ptr(ip, ctx->params.hashLog3)];
if (matchIndex3 < current && matchIndex3 >= lowLimit) {
size_t offset = (size_t)current - matchIndex3;
if (offset < LZ5_MAX_8BIT_OFFSET) {
match = ip - offset;
if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) {
mlt = LZ5_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH;
int back = 0;
while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--;
mlt -= back;
if (!longest || (mlt > longest && LZ5_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off))) {
*matchpos = match + back;
*startpos = ip + back;
longest = mlt;
}
}
}
}
}
#endif
while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) {
nbAttempts--;
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_LOWESTPRICE_MIN_OFFSET) {
if (matchIndex >= dictLimit) {
if (MEM_read32(match) == MEM_read32(ip)) {
int back = 0;
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH;
while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (!longest || (mlt > longest && LZ5_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (ip - match), mlt, ctx->last_off)))
{ longest = mlt; *startpos = ip+back; *matchpos = match+back; }
}
} else {
matchDict = dictBase + matchIndex;
// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match));
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(matchDict) == MEM_read32(ip)) {
int back=0;
mlt = LZ5_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH;
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--;
mlt -= back;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (!longest || (mlt > longest && LZ5_better_price(ctx, *startpos, (*startpos - *matchpos), longest, ip, (U32)(ip - match), mlt, ctx->last_off)))
{ longest = mlt; *startpos = ip+back; *matchpos = match+back; } /* virtual matchpos */
}
}
}
matchIndex -= chainTable[matchIndex & contentMask];
}
return longest;
}
FORCE_INLINE int LZ5_compress_lowestPrice(
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const BYTE* anchor = ip;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
size_t ml, ml2, ml0;
const BYTE* ref=NULL;
const BYTE* start2=NULL;
const BYTE* ref2=NULL;
const BYTE* start0;
const BYTE* ref0;
const BYTE* lowPrefixPtr = ctx->base + ctx->dictLimit;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
const size_t sufficient_len = ctx->params.sufficientLength;
/* Main Loop */
while (ip < mflimit)
{
LZ5_Insert(ctx, ip);
ml = LZ5_FindMatchLowestPrice (ctx, ip, matchlimit, (&ref));
if (!ml) { ip++; continue; }
{
int back = 0;
while ((ip + back > anchor) && (ref + back > lowPrefixPtr) && (ip[back - 1] == ref[back - 1])) back--;
ml -= back;
ip += back;
ref += back;
}
/* saved, in case we would skip too much */
start0 = ip;
ref0 = ref;
ml0 = ml;
// goto _Encode;
_Search:
if (ip+ml >= mflimit) { goto _Encode; }
if (ml >= sufficient_len) { goto _Encode; }
LZ5_Insert(ctx, ip);
ml2 = (int)LZ5_GetWiderMatch(ctx, ip + ml - 2, anchor, matchlimit, 0, &ref2, &start2);
if (!ml2) goto _Encode;
{
U64 price, best_price;
int off0=0, off1=0;
const BYTE *pos, *best_pos;
// find the lowest price for encoding ml bytes
best_pos = ip;
best_price = LZ5_MAX_PRICE;
off0 = (int)(ip - ref);
off1 = (int)(start2 - ref2);
for (pos = ip + ml; pos >= start2; pos--)
{
int common0 = (int)(pos - ip);
if (common0 >= MINMATCH) {
price = (int)LZ5_get_price_LZ5v2(ctx, ctx->last_off, ip, ctx->off24pos, ip - anchor, (off0 == ctx->last_off) ? 0 : off0, common0);
{
int common1 = (int)(start2 + ml2 - pos);
if (common1 >= MINMATCH)
price += LZ5_get_price_LZ5v2(ctx, ctx->last_off, pos, ctx->off24pos, 0, (off1 == off0) ? 0 : (off1), common1);
else
price += LZ5_get_price_LZ5v2(ctx, ctx->last_off, pos, ctx->off24pos, common1, 0, 0);
}
if (price < best_price) {
best_price = price;
best_pos = pos;
}
} else {
price = LZ5_get_price_LZ5v2(ctx, ctx->last_off, ip, ctx->off24pos, start2 - anchor, (off1 == ctx->last_off) ? 0 : off1, ml2);
if (price < best_price)
best_pos = pos;
break;
}
}
// LZ5_DEBUG("%u: TRY last_off=%d literals=%u off=%u mlen=%u literals2=%u off2=%u mlen2=%u best=%d\n", (U32)(ip - ctx->inputBuffer), ctx->last_off, (U32)(ip - anchor), off0, (U32)ml, (U32)(start2 - anchor), off1, ml2, (U32)(best_pos - ip));
ml = (int)(best_pos - ip);
}
if ((ml < MINMATCH) || ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LZ5_MAX_16BIT_OFFSET)))
{
ip = start2;
ref = ref2;
ml = ml2;
goto _Search;
}
_Encode:
if (start0 < ip)
{
if (LZ5_more_profitable(ctx, ip, (ip - ref), ml, start0, (start0 - ref0), ml0, (ref0 - ref), ctx->last_off))
{
ip = start0;
ref = ref0;
ml = ml0;
}
}
// if ((ml < minMatchLongOff) && ((U32)(ip-ref) >= LZ5_MAX_16BIT_OFFSET)) { printf("LZ5_encodeSequence ml=%d off=%d\n", ml, (U32)(ip-ref)); exit(0); }
if (LZ5_encodeSequence_LZ5v2(ctx, &ip, &anchor, ml, ((ip - ref == ctx->last_off) ? ip : ref))) return 0;
}
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ5v2(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_nochain.h
+318
View File
@@ -0,0 +1,318 @@
#define OPTIMAL_ML (int)((ML_MASK_LZ4-1)+MINMATCH)
//#define LZ5_NOCHAIN_HASH_FUNCTION(ip, hashLog) LZ5_hashPtr(ip, hashLog, ctx->params.searchLength)
#define LZ5_NOCHAIN_HASH_FUNCTION(ip, hashLog) LZ5_hash5Ptr(ip, hashLog)
#define LZ5_NOCHAIN_MIN_OFFSET 8
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ5_InsertNoChain (LZ5_stream_t* ctx, const BYTE* ip)
{
U32* const hashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
U32 const target = (U32)(ip - base);
U32 idx = ctx->nextToUpdate;
const int hashLog = ctx->params.hashLog;
while (idx < target) {
size_t const h = LZ5_NOCHAIN_HASH_FUNCTION(base+idx, hashLog);
if (idx >= hashTable[h] + LZ5_NOCHAIN_MIN_OFFSET)
hashTable[h] = idx;
idx++;
}
ctx->nextToUpdate = target;
}
FORCE_INLINE int LZ5_InsertAndFindBestMatchNoChain (LZ5_stream_t* ctx, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos)
{
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
U32 matchIndex;
const BYTE* match;
size_t ml=0;
const int hashLog = ctx->params.hashLog;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance;
/* HC4 match finder */
LZ5_InsertNoChain(ctx, ip);
matchIndex = HashTable[LZ5_NOCHAIN_HASH_FUNCTION(ip, hashLog)];
if (matchIndex >= lowLimit) {
if (matchIndex >= dictLimit) {
match = base + matchIndex;
#if LZ5_NOCHAIN_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_NOCHAIN_MIN_OFFSET)
#endif
if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip)))
{
size_t const mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
} else {
match = dictBase + matchIndex;
// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match));
#if LZ5_NOCHAIN_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_NOCHAIN_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
size_t mlt = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
}
return (int)ml;
}
FORCE_INLINE int LZ5_InsertAndGetWiderMatchNoChain (
LZ5_stream_t* ctx,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
int longest,
const BYTE** matchpos,
const BYTE** startpos)
{
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
U32 matchIndex;
int LLdelta = (int)(ip-iLowLimit);
const int hashLog = ctx->params.hashLog;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 lowLimit = (ctx->lowLimit + maxDistance >= (U32)(ip-base)) ? ctx->lowLimit : (U32)(ip - base) - maxDistance;
/* First Match */
LZ5_InsertNoChain(ctx, ip);
matchIndex = HashTable[LZ5_NOCHAIN_HASH_FUNCTION(ip, hashLog)];
if (matchIndex>=lowLimit) {
if (matchIndex >= dictLimit) {
const BYTE* match = base + matchIndex;
#if LZ5_NOCHAIN_MIN_OFFSET > 0
if ((U32)(ip - match) >= LZ5_NOCHAIN_MIN_OFFSET)
#endif
if (*(iLowLimit + longest) == *(match - LLdelta + longest)) {
if (MEM_read32(match) == MEM_read32(ip)) {
int mlt = MINMATCH + LZ5_count(ip+MINMATCH, match+MINMATCH, iHighLimit);
int back = 0;
while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if (mlt > longest) {
longest = (int)mlt;
*matchpos = match+back;
*startpos = ip+back;
}
}
}
} else {
const BYTE* match = dictBase + matchIndex;
#if LZ5_NOCHAIN_MIN_OFFSET > 0
if ((U32)(ip - (base + matchIndex)) >= LZ5_NOCHAIN_MIN_OFFSET)
#endif
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(match) == MEM_read32(ip)) {
int back=0;
size_t mlt = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH;
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
}
return longest;
}
FORCE_INLINE int LZ5_compress_noChain (
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const BYTE* anchor = ip;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
int ml, ml2, ml3, ml0;
const BYTE* ref = NULL;
const BYTE* start2 = NULL;
const BYTE* ref2 = NULL;
const BYTE* start3 = NULL;
const BYTE* ref3 = NULL;
const BYTE* start0;
const BYTE* ref0;
/* init */
ip++;
/* Main Loop */
while (ip < mflimit) {
ml = LZ5_InsertAndFindBestMatchNoChain (ctx, ip, matchlimit, (&ref));
if (!ml) { ip++; continue; }
/* saved, in case we would skip too much */
start0 = ip;
ref0 = ref;
ml0 = ml;
_Search2:
if (ip+ml < mflimit)
ml2 = LZ5_InsertAndGetWiderMatchNoChain(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2);
else ml2 = ml;
if (ml2 == ml) { /* No better match */
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
continue;
}
if (start0 < ip) {
if (start2 < ip + ml0) { /* empirical */
ip = start0;
ref = ref0;
ml = ml0;
}
}
/* Here, start0==ip */
if ((start2 - ip) < 3) { /* First Match too small : removed */
ml = ml2;
ip = start2;
ref =ref2;
goto _Search2;
}
_Search3:
/*
* Currently we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
if ((start2 - ip) < OPTIMAL_ML) {
int correction;
int new_ml = ml;
if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
correction = new_ml - (int)(start2 - ip);
if (correction > 0) {
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
}
/* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
if (start2 + ml2 < mflimit)
ml3 = LZ5_InsertAndGetWiderMatchNoChain(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3);
else ml3 = ml2;
if (ml3 == ml2) { /* No better match : 2 sequences to encode */
/* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start2;
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml2, ref2)) return 0;
continue;
}
if (start3 < ip+ml+3) { /* Not enough space for match 2 : remove it */
if (start3 >= (ip+ml)) { /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
if (start2 < ip+ml) {
int correction = (int)(ip+ml - start2);
start2 += correction;
ref2 += correction;
ml2 -= correction;
if (ml2 < MINMATCH) {
start2 = start3;
ref2 = ref3;
ml2 = ml3;
}
}
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start3;
ref = ref3;
ml = ml3;
start0 = start2;
ref0 = ref2;
ml0 = ml2;
goto _Search2;
}
start2 = start3;
ref2 = ref3;
ml2 = ml3;
goto _Search3;
}
/*
* OK, now we have 3 ascending matches; let's write at least the first one
* ip & ref are known; Now for ml
*/
if (start2 < ip+ml) {
if ((start2 - ip) < (int)ML_MASK_LZ4) {
int correction;
if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
if (ip + ml > start2 + ml2 - MINMATCH) {
ml = (int)(start2 - ip) + ml2 - MINMATCH;
if (ml < MINMATCH) { // match2 doesn't fit, remove it
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start3;
ref = ref3;
ml = ml3;
start0 = start2;
ref0 = ref2;
ml0 = ml2;
goto _Search2;
}
}
correction = ml - (int)(start2 - ip);
if (correction > 0) {
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
} else {
ml = (int)(start2 - ip);
}
}
if (LZ5_encodeSequence_LZ4(ctx, &ip, &anchor, ml, ref)) return 0;
ip = start2;
ref = ref2;
ml = ml2;
start2 = start3;
ref2 = ref3;
ml2 = ml3;
goto _Search3;
}
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ4(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_optimal.h
+679
View File
@@ -0,0 +1,679 @@
#define LZ5_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ5_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ5_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ5_OPTIMAL_MIN_OFFSET 8
#define LZ5_OPT_NUM (1<<12)
#define REPMINMATCH 1
FORCE_INLINE size_t LZ5_get_price(LZ5_stream_t* const ctx, int rep, const BYTE *ip, const BYTE *off24pos, size_t litLength, U32 offset, size_t matchLength)
{
if (ctx->params.decompressType == LZ5_coderwords_LZ4)
return LZ5_get_price_LZ4(ctx, ip, litLength, offset, matchLength);
return LZ5_get_price_LZ5v2(ctx, rep, ip, off24pos, litLength, offset, matchLength);
}
typedef struct
{
int off;
int len;
int back;
} LZ5_match_t;
typedef struct
{
int price;
int off;
int mlen;
int litlen;
int rep;
const BYTE* off24pos;
} LZ5_optimal_t;
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ5_BinTree_Insert(LZ5_stream_t* ctx, const BYTE* ip)
{
#if MINMATCH == 3
U32* HashTable3 = ctx->hashTable3;
const BYTE* const base = ctx->base;
const U32 target = (U32)(ip - base);
U32 idx = ctx->nextToUpdate;
while(idx < target) {
HashTable3[LZ5_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx;
idx++;
}
ctx->nextToUpdate = target;
#else
(void)ctx; (void)ip;
#endif
}
FORCE_INLINE int LZ5_GetAllMatches (
LZ5_stream_t* ctx,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
size_t best_mlen,
LZ5_match_t* matches)
{
U32* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const intptr_t dictLimit = ctx->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1;
const intptr_t current = (ip - base);
const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const BYTE* match, *matchDict;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
intptr_t matchIndex;
int nbAttempts = ctx->params.searchNum;
// bool fullSearch = (ctx->params.fullSearch >= 2);
int mnum = 0;
U32* HashPos;
size_t mlt;
if (ip + MINMATCH > iHighLimit) return 0;
/* First Match */
HashPos = &HashTable[LZ5_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)];
matchIndex = *HashPos;
#if MINMATCH == 3
{
U32* const HashTable3 = ctx->hashTable3;
U32* HashPos3 = &HashTable3[LZ5_hash3Ptr(ip, ctx->params.hashLog3)];
if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) {
size_t offset = current - *HashPos3;
if (offset < LZ5_MAX_8BIT_OFFSET) {
match = ip - offset;
if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) {
size_t mlt = LZ5_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH;
int back = 0;
while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--;
mlt -= back;
matches[mnum].off = (int)offset;
matches[mnum].len = (int)mlt;
matches[mnum].back = -back;
mnum++;
}
}
}
*HashPos3 = current;
}
#endif
chainTable[current & contentMask] = (U32)(current - matchIndex);
*HashPos = (U32)current;
ctx->nextToUpdate++;
if (best_mlen < MINMATCH-1) best_mlen = MINMATCH-1;
while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) {
nbAttempts--;
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_OPTIMAL_MIN_OFFSET) {
if (matchIndex >= dictLimit) {
if ((/*fullSearch ||*/ ip[best_mlen] == match[best_mlen]) && (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip))) {
int back = 0;
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH;
while ((ip+back > iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--;
mlt -= back;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (mlt > best_mlen) {
best_mlen = mlt;
matches[mnum].off = (int)(ip - match);
matches[mnum].len = (int)mlt;
matches[mnum].back = -back;
mnum++;
if (best_mlen > LZ5_OPT_NUM) break;
}
}
} else {
matchDict = dictBase + matchIndex;
// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match));
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_readMINMATCH(matchDict) == MEM_readMINMATCH(ip)) {
int back=0;
mlt = LZ5_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iHighLimit, dictEnd, lowPrefixPtr) + MINMATCH;
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchDict[back-1])) back--;
mlt -= back;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (mlt > best_mlen) {
best_mlen = mlt;
matches[mnum].off = (int)(ip - match);
matches[mnum].len = (int)mlt;
matches[mnum].back = -back;
mnum++;
if (best_mlen > LZ5_OPT_NUM) break;
}
}
}
}
matchIndex -= chainTable[matchIndex & contentMask];
}
return mnum;
}
FORCE_INLINE int LZ5_BinTree_GetAllMatches (
LZ5_stream_t* ctx,
const BYTE* const ip,
const BYTE* const iHighLimit,
size_t best_mlen,
LZ5_match_t* matches)
{
U32* const chainTable = ctx->chainTable;
U32* const HashTable = ctx->hashTable;
const BYTE* const base = ctx->base;
const intptr_t dictLimit = ctx->dictLimit;
const BYTE* const dictBase = ctx->dictBase;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 contentMask = (1 << ctx->params.contentLog) - 1;
const BYTE* const lowPrefixPtr = base + dictLimit;
const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1;
const intptr_t current = (ip - base);
const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance;
const BYTE* match;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
int nbAttempts = ctx->params.searchNum;
int mnum = 0;
U32 *ptr0, *ptr1, delta0, delta1;
intptr_t matchIndex;
size_t mlt = 0;
U32* HashPos;
if (ip + MINMATCH > iHighLimit) return 0;
/* First Match */
HashPos = &HashTable[LZ5_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)];
matchIndex = *HashPos;
#if MINMATCH == 3
{
U32* HashPos3 = &ctx->hashTable3[LZ5_hash3Ptr(ip, ctx->params.hashLog3)];
if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) {
size_t offset = current - *HashPos3;
if (offset < LZ5_MAX_8BIT_OFFSET) {
match = ip - offset;
if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match))
{
mlt = LZ5_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH;
matches[mnum].off = (int)offset;
matches[mnum].len = (int)mlt;
matches[mnum].back = 0;
mnum++;
}
}
*HashPos3 = current;
}
}
#endif
*HashPos = (U32)current;
ctx->nextToUpdate++;
// check rest of matches
ptr0 = &chainTable[(current*2+1) & contentMask];
ptr1 = &chainTable[(current*2) & contentMask];
delta0 = delta1 = (U32)(current - matchIndex);
if (best_mlen < MINMATCH-1) best_mlen = MINMATCH-1;
while ((matchIndex < current) && (matchIndex >= lowLimit) && (nbAttempts)) {
nbAttempts--;
if (matchIndex >= dictLimit) {
match = base + matchIndex;
// if (ip[mlt] == match[mlt])
mlt = LZ5_count(ip, match, iHighLimit);
} else {
match = dictBase + matchIndex;
mlt = LZ5_count_2segments(ip, match, iHighLimit, dictEnd, lowPrefixPtr);
if (matchIndex + (int)mlt >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[mlt] */
}
if ((U32)(current - matchIndex) >= LZ5_OPTIMAL_MIN_OFFSET) {
if ((mlt >= minMatchLongOff) || ((U32)(current - matchIndex) < LZ5_MAX_16BIT_OFFSET))
if (mlt > best_mlen) {
best_mlen = mlt;
matches[mnum].off = (int)(current - matchIndex);
matches[mnum].len = (int)mlt;
matches[mnum].back = 0;
mnum++;
if (mlt > LZ5_OPT_NUM) break;
if (ip + mlt >= iHighLimit) break;
}
} else {
#if 1
intptr_t newMatchIndex;
size_t newml = 0, newoff = 0;
do {
newoff += (int)(current - matchIndex);
} while (newoff < LZ5_OPTIMAL_MIN_OFFSET);
newMatchIndex = current - newoff;
if (newMatchIndex >= dictLimit) newml = LZ5_count(ip, base + newMatchIndex, iHighLimit);
// printf("%d: off=%d mlt=%d\n", (U32)current, (U32)(current - matchIndex), (int)mlt);
// printf("%d: newoff=%d newml=%d\n", (U32)current, (int)newoff, (int)newml);
if ((newml >= minMatchLongOff) && (newml > best_mlen)) {
best_mlen = newml;
matches[mnum].off = (int)newoff;
matches[mnum].len = (int)newml;
matches[mnum].back = 0;
mnum++;
if (newml > LZ5_OPT_NUM) break;
if (ip + newml >= iHighLimit) break;
}
#endif
}
if (ip[mlt] < match[mlt]) {
*ptr0 = delta0;
ptr0 = &chainTable[(matchIndex*2) & contentMask];
if (*ptr0 == (U32)-1) break;
delta0 = *ptr0;
delta1 += delta0;
matchIndex -= delta0;
} else {
*ptr1 = delta1;
ptr1 = &chainTable[(matchIndex*2+1) & contentMask];
if (*ptr1 == (U32)-1) break;
delta1 = *ptr1;
delta0 += delta1;
matchIndex -= delta1;
}
}
*ptr0 = (U32)-1;
*ptr1 = (U32)-1;
return mnum;
}
#define SET_PRICE(pos, mlen, offset, litlen, price) \
{ \
while (last_pos < pos) { opt[last_pos+1].price = LZ5_MAX_PRICE; last_pos++; } \
opt[pos].mlen = (int)mlen; \
opt[pos].off = (int)offset; \
opt[pos].litlen = (int)litlen; \
opt[pos].price = (int)price; \
LZ5_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \
}
FORCE_INLINE int LZ5_compress_optimalPrice(
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
LZ5_optimal_t opt[LZ5_OPT_NUM + 4];
LZ5_match_t matches[LZ5_OPT_NUM + 1];
const BYTE *inr;
size_t res, cur, cur2, skip_num = 0;
size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos;
const BYTE* anchor = ip;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const intptr_t dictLimit = ctx->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const intptr_t lowLimit = ctx->lowLimit;
const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1;
const size_t sufficient_len = ctx->params.sufficientLength;
const int faster_get_matches = (ctx->params.fullSearch == 0);
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
const int lz5OptimalMinOffset = (ctx->params.decompressType == LZ5_coderwords_LZ4) ? (1<<30) : LZ5_OPTIMAL_MIN_OFFSET;
const size_t repMinMatch = (ctx->params.decompressType == LZ5_coderwords_LZ4) ? MINMATCH : REPMINMATCH;
/* Main Loop */
while (ip < mflimit) {
memset(opt, 0, sizeof(LZ5_optimal_t));
last_pos = 0;
llen = ip - anchor;
/* check rep code */
if (ctx->last_off >= lz5OptimalMinOffset) {
intptr_t matchIndexLO = (ip - ctx->last_off) - base;
mlen = 0;
if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= ip)) {
if (matchIndexLO >= dictLimit) {
mlen = LZ5_count(ip, base + matchIndexLO, matchlimit);
} else {
mlen = LZ5_count_2segments(ip, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr);
}
}
if (mlen >= REPMINMATCH) {
if (mlen > sufficient_len || mlen >= LZ5_OPT_NUM) {
best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1;
goto encode;
}
do
{
litlen = 0;
price = LZ5_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen, 0, mlen);
if (mlen > last_pos || price < (size_t)opt[mlen].price)
SET_PRICE(mlen, mlen, 0, litlen, price);
mlen--;
}
while (mlen >= REPMINMATCH);
}
}
if (faster_get_matches && last_pos)
match_num = 0;
else
{
if (ctx->params.parserType == LZ5_parser_optimalPrice) {
LZ5_Insert(ctx, ip);
match_num = LZ5_GetAllMatches(ctx, ip, ip, matchlimit, last_pos, matches);
} else {
LZ5_BinTree_Insert(ctx, ip);
match_num = LZ5_BinTree_GetAllMatches(ctx, ip, matchlimit, last_pos, matches);
}
}
LZ5_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos);
if (!last_pos && !match_num) { ip++; continue; }
if (match_num && (size_t)matches[match_num-1].len > sufficient_len) {
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
cur = 0;
last_pos = 1;
goto encode;
}
// set prices using matches at position = 0
best_mlen = (last_pos > MINMATCH) ? last_pos : MINMATCH;
for (i = 0; i < match_num; i++) {
mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen;
best_mlen = (matches[i].len < LZ5_OPT_NUM) ? matches[i].len : LZ5_OPT_NUM;
LZ5_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos);
while (mlen <= best_mlen){
litlen = 0;
price = LZ5_get_price(ctx, ctx->last_off, ip, ctx->off24pos, llen + litlen, matches[i].off, mlen);
if ((mlen >= minMatchLongOff) || (matches[i].off < LZ5_MAX_16BIT_OFFSET))
if (mlen > last_pos || price < (size_t)opt[mlen].price)
SET_PRICE(mlen, mlen, matches[i].off, litlen, price);
mlen++;
}
}
if (last_pos < repMinMatch) { ip++; continue; }
opt[0].off24pos = ctx->off24pos;
opt[0].rep = ctx->last_off;
opt[0].mlen = 1;
opt[0].off = -1;
// check further positions
for (skip_num = 0, cur = 1; cur <= last_pos; cur++) {
int rep;
inr = ip + cur;
if (opt[cur-1].off == -1) { // -1 = literals, 0 = rep
litlen = opt[cur-1].litlen + 1;
if (cur != litlen) {
price = opt[cur - litlen].price + LZ5_get_price(ctx, opt[cur-litlen].rep, inr, ctx->off24pos, litlen, 0, 0);
LZ5_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
} else {
price = LZ5_get_price(ctx, ctx->last_off, inr, ctx->off24pos, llen + litlen, 0, 0);
LZ5_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen);
}
} else {
litlen = 1;
price = opt[cur - 1].price + LZ5_get_price(ctx, opt[cur-1].rep, inr, ctx->off24pos, litlen, 0, 0);
LZ5_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ5_get_price(ctx, rep, inr, ctx->off24pos, litlen, 0, 0));
}
mlen = 1;
best_mlen = 0;
LZ5_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price);
if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen)))
SET_PRICE(cur, mlen, -1, litlen, price);
if (cur == last_pos) break;
/* set rep code */
if (opt[cur].off != -1) {
mlen = opt[cur].mlen;
offset = opt[cur].off;
if (offset < 1) {
opt[cur].rep = opt[cur-mlen].rep;
opt[cur].off24pos = opt[cur-mlen].off24pos;
LZ5_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d\n", (int)(inr-source), cur, mlen, opt[cur-mlen].rep);
} else {
opt[cur].rep = (int)offset;
opt[cur].off24pos = (offset >= LZ5_MAX_16BIT_OFFSET) ? inr : opt[cur-mlen].off24pos;
LZ5_LOG_PARSER("%d: COPYREP2 cur=%d offset=%d rep=%d\n", (int)(inr-source), cur, offset, opt[cur].rep);
}
} else {
opt[cur].rep = opt[cur-1].rep; // copy rep
opt[cur].off24pos = opt[cur-1].off24pos;
}
rep = opt[cur].rep;
LZ5_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep);
/* check rep code */
if (opt[cur].rep >= lz5OptimalMinOffset) {
intptr_t matchIndexLO = (inr - opt[cur].rep) - base;
mlen = 0;
if ((matchIndexLO >= lowLimit) && (base + matchIndexLO + maxDistance >= inr)) {
if (matchIndexLO >= dictLimit) {
mlen = LZ5_count(inr, base + matchIndexLO, matchlimit);
} else {
mlen = LZ5_count_2segments(inr, dictBase + matchIndexLO, matchlimit, dictEnd, lowPrefixPtr);
}
}
if (mlen >= REPMINMATCH/* && mlen > best_mlen*/) {
LZ5_LOG_PARSER("%d: try REP rep=%d mlen=%d\n", (int)(inr-source), opt[cur].rep, mlen);
LZ5_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-source), mlen, 0, opt[cur].rep, cur, opt[cur].off);
if (mlen > sufficient_len || cur + mlen >= LZ5_OPT_NUM) {
best_mlen = mlen;
best_off = 0;
LZ5_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-source), sufficient_len, best_mlen, best_off, last_pos);
last_pos = cur + 1;
goto encode;
}
best_mlen = mlen;
if (faster_get_matches)
skip_num = best_mlen;
do
{
//if (opt[cur].mlen == 1)
if (opt[cur].off == -1) {
litlen = opt[cur].litlen;
if (cur != litlen) {
price = opt[cur - litlen].price + LZ5_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen);
LZ5_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen);
} else {
price = LZ5_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, 0, mlen);
LZ5_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen);
}
} else {
litlen = 0;
price = opt[cur].price + LZ5_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen);
LZ5_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d getprice=%d\n", (int)(inr-source), price, cur, litlen, LZ5_get_price(ctx, rep, inr, opt[cur].off24pos, litlen, 0, mlen - MINMATCH));
}
LZ5_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price);
if (cur + mlen > last_pos || price <= (size_t)opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH
SET_PRICE(cur + mlen, mlen, 0, litlen, price);
mlen--;
}
while (mlen >= REPMINMATCH);
}
}
if (faster_get_matches && skip_num > 0) {
skip_num--;
continue;
}
if (ctx->params.parserType == LZ5_parser_optimalPrice) {
LZ5_Insert(ctx, inr);
match_num = LZ5_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches);
LZ5_LOG_PARSER("%d: LZ5_GetAllMatches match_num=%d\n", (int)(inr-source), match_num);
} else {
LZ5_BinTree_Insert(ctx, inr);
match_num = LZ5_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches);
LZ5_LOG_PARSER("%d: LZ5_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num);
}
if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) {
cur -= matches[match_num-1].back;
best_mlen = matches[match_num-1].len;
best_off = matches[match_num-1].off;
last_pos = cur + 1;
goto encode;
}
// set prices using matches at position = cur
best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH;
for (i = 0; i < match_num; i++) {
mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen;
cur2 = cur - matches[i].back;
best_mlen = (cur2 + matches[i].len < LZ5_OPT_NUM) ? (size_t)matches[i].len : LZ5_OPT_NUM - cur2;
LZ5_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos);
if (mlen < (size_t)matches[i].back + 1)
mlen = matches[i].back + 1;
while (mlen <= best_mlen) {
// if (opt[cur2].mlen == 1)
if (opt[cur2].off == -1)
{
litlen = opt[cur2].litlen;
if (cur2 != litlen)
price = opt[cur2 - litlen].price + LZ5_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen);
else
price = LZ5_get_price(ctx, rep, inr, ctx->off24pos, llen + litlen, matches[i].off, mlen);
} else {
litlen = 0;
price = opt[cur2].price + LZ5_get_price(ctx, rep, inr, opt[cur2].off24pos, litlen, matches[i].off, mlen);
}
LZ5_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price);
// if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price)))
if ((mlen >= minMatchLongOff) || (matches[i].off < LZ5_MAX_16BIT_OFFSET))
if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price)
{
SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price);
}
mlen++;
}
}
} // for (skip_num = 0, cur = 1; cur <= last_pos; cur++)
best_mlen = opt[last_pos].mlen;
best_off = opt[last_pos].off;
cur = last_pos - best_mlen;
encode: // cur, last_pos, best_mlen, best_off have to be set
for (i = 1; i <= last_pos; i++) {
LZ5_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep);
}
LZ5_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off, opt[cur].rep);
opt[0].mlen = 1;
while (1) {
mlen = opt[cur].mlen;
offset = opt[cur].off;
opt[cur].mlen = (int)best_mlen;
opt[cur].off = (int)best_off;
best_mlen = mlen;
best_off = offset;
if (mlen > cur) break;
cur -= mlen;
}
for (i = 0; i <= last_pos;) {
LZ5_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep);
i += opt[i].mlen;
}
cur = 0;
while (cur < last_pos) {
LZ5_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep);
mlen = opt[cur].mlen;
// if (mlen == 1) { ip++; cur++; continue; }
if (opt[cur].off == -1) { ip++; cur++; continue; }
offset = opt[cur].off;
cur += mlen;
LZ5_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen);
res = LZ5_encodeSequence(ctx, &ip, &anchor, mlen, ip - offset);
if (res) return 0;
LZ5_LOG_PARSER("%d: offset=%d rep=%d\n", (int)(ip-source), offset, ctx->last_off);
}
}
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5_parser_pricefast.h
+253
View File
@@ -0,0 +1,253 @@
#define LZ5_PRICEFAST_MIN_OFFSET 8
FORCE_INLINE int LZ5_FindMatchFast(LZ5_stream_t* ctx, intptr_t matchIndex, intptr_t matchIndex3, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos)
{
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const intptr_t dictLimit = ctx->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const intptr_t maxDistance = (1 << ctx->params.windowLog) - 1;
const intptr_t current = (U32)(ip - base);
const intptr_t lowLimit = ((intptr_t)ctx->lowLimit + maxDistance >= current) ? (intptr_t)ctx->lowLimit : current - maxDistance;
const BYTE* const lowPrefixPtr = base + dictLimit;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
const BYTE* match, *matchDict;
size_t ml=0, mlt;
if (ctx->last_off >= LZ5_PRICEFAST_MIN_OFFSET) {
intptr_t matchIndexLO = (ip - ctx->last_off) - base;
if (matchIndexLO >= lowLimit) {
if (matchIndexLO >= dictLimit) {
match = base + matchIndexLO;
if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) {
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
// if ((mlt >= minMatchLongOff) || (ctx->last_off < LZ5_MAX_16BIT_OFFSET))
{
*matchpos = match;
return (int)mlt;
}
}
} else {
match = dictBase + matchIndexLO;
if ((U32)((dictLimit-1) - matchIndexLO) >= 3) /* intentional overflow */
if (MEM_readMINMATCH(match) == MEM_readMINMATCH(ip)) {
mlt = LZ5_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH;
// if ((mlt >= minMatchLongOff) || (ctx->last_off < LZ5_MAX_16BIT_OFFSET))
{
*matchpos = base + matchIndexLO; /* virtual matchpos */
return (int)mlt;
}
}
}
}
}
#if MINMATCH == 3
if (matchIndex3 < current && matchIndex3 >= lowLimit) {
intptr_t offset = current - matchIndex3;
if (offset < LZ5_MAX_8BIT_OFFSET) {
match = ip - offset;
if (match > base && MEM_readMINMATCH(ip) == MEM_readMINMATCH(match)) {
ml = 3;//LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
*matchpos = match;
}
}
}
#else
(void)matchIndex3;
#endif
if ((matchIndex < current) && (matchIndex >= lowLimit)) {
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_PRICEFAST_MIN_OFFSET) {
if (matchIndex >= dictLimit) {
if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) {
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (!ml || (mlt > ml)) // && LZ5_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off)))
{ ml = mlt; *matchpos = match; }
}
} else {
matchDict = dictBase + matchIndex;
// fprintf(stderr, "dictBase[%p]+matchIndex[%d]=match[%p] dictLimit=%d base=%p ip=%p iLimit=%p off=%d\n", dictBase, matchIndex, match, dictLimit, base, ip, iLimit, (U32)(ip-match));
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(matchDict) == MEM_read32(ip)) {
mlt = LZ5_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
if (!ml || (mlt > ml)) // && LZ5_better_price((ip - *matchpos), ml, (U32)(ip - match), mlt, ctx->last_off)))
{ ml = mlt; *matchpos = match; } /* virtual matchpos */
}
}
}
}
return (int)ml;
}
FORCE_INLINE int LZ5_FindMatchFaster (LZ5_stream_t* ctx, U32 matchIndex, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos)
{
const BYTE* const base = ctx->base;
const BYTE* const dictBase = ctx->dictBase;
const U32 dictLimit = ctx->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const U32 maxDistance = (1 << ctx->params.windowLog) - 1;
const U32 current = (U32)(ip - base);
const U32 lowLimit = (ctx->lowLimit + maxDistance >= current) ? ctx->lowLimit : current - maxDistance;
const BYTE* const lowPrefixPtr = base + dictLimit;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
const BYTE* match, *matchDict;
size_t ml=0, mlt;
if (matchIndex < current && matchIndex >= lowLimit) {
match = base + matchIndex;
if ((U32)(ip - match) >= LZ5_PRICEFAST_MIN_OFFSET) {
if (matchIndex >= dictLimit) {
if (MEM_read32(match) == MEM_read32(ip)) {
mlt = LZ5_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
{ ml = mlt; *matchpos = match; }
}
} else {
matchDict = dictBase + matchIndex;
if ((U32)((dictLimit-1) - matchIndex) >= 3) /* intentional overflow */
if (MEM_read32(matchDict) == MEM_read32(ip)) {
mlt = LZ5_count_2segments(ip+MINMATCH, matchDict+MINMATCH, iLimit, dictEnd, lowPrefixPtr) + MINMATCH;
if ((mlt >= minMatchLongOff) || ((U32)(ip - match) < LZ5_MAX_16BIT_OFFSET))
{ ml = mlt; *matchpos = match; } /* virtual matchpos */
}
}
}
}
return (int)ml;
}
FORCE_INLINE int LZ5_compress_priceFast(
LZ5_stream_t* const ctx,
const BYTE* ip,
const BYTE* const iend)
{
const BYTE* anchor = ip;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
size_t ml, ml2=0;
const BYTE* ref=NULL;
const BYTE* start2=NULL;
const BYTE* ref2=NULL;
const BYTE* lowPrefixPtr = ctx->base + ctx->dictLimit;
U32* HashTable = ctx->hashTable;
#if MINMATCH == 3
U32* HashTable3 = ctx->hashTable3;
#endif
const BYTE* const base = ctx->base;
const size_t minMatchLongOff = ctx->params.minMatchLongOff;
U32* HashPos;
/* init */
ip++;
/* Main Loop */
while (ip < mflimit)
{
HashPos = &HashTable[LZ5_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)];
#if MINMATCH == 3
{
U32* HashPos3 = &HashTable3[LZ5_hash3Ptr(ip, ctx->params.hashLog3)];
ml = LZ5_FindMatchFast (ctx, *HashPos, *HashPos3, ip, matchlimit, (&ref));
*HashPos3 = (U32)(ip - base);
}
#else
ml = LZ5_FindMatchFast (ctx, *HashPos, 0, ip, matchlimit, (&ref));
#endif
if ((U32)(ip - base) >= *HashPos + LZ5_PRICEFAST_MIN_OFFSET)
*HashPos = (U32)(ip - base);
if (!ml) { ip++; continue; }
if ((int)(ip - ref) == ctx->last_off) { ml2=0; ref=ip; goto _Encode; }
{
int back = 0;
while ((ip+back>anchor) && (ref+back > lowPrefixPtr) && (ip[back-1] == ref[back-1])) back--;
ml -= back;
ip += back;
ref += back;
}
_Search:
if (ip+ml >= mflimit) goto _Encode;
start2 = ip + ml - 2;
HashPos = &HashTable[LZ5_hashPtr(start2, ctx->params.hashLog, ctx->params.searchLength)];
ml2 = LZ5_FindMatchFaster(ctx, *HashPos, start2, matchlimit, (&ref2));
if ((U32)(start2 - base) >= *HashPos + LZ5_PRICEFAST_MIN_OFFSET)
*HashPos = (U32)(start2 - base);
if (!ml2) goto _Encode;
{
int back = 0;
while ((start2+back>ip) && (ref2+back > lowPrefixPtr) && (start2[back-1] == ref2[back-1])) back--;
ml2 -= back;
start2 += back;
ref2 += back;
}
// LZ5_DEBUG("%u: TRY last_off=%d literals=%u off=%u mlen=%u literals2=%u off2=%u mlen2=%u best=%d\n", (U32)(ip - ctx->inputBuffer), ctx->last_off, (U32)(ip - anchor), off0, (U32)ml, (U32)(start2 - anchor), off1, ml2, (U32)(best_pos - ip));
if (ml2 <= ml) { ml2 = 0; goto _Encode; }
if (start2 <= ip)
{
ip = start2; ref = ref2; ml = ml2;
ml2 = 0;
goto _Encode;
}
if (start2 - ip < 3)
{
ip = start2; ref = ref2; ml = ml2;
ml2 = 0;
goto _Search;
}
if (start2 < ip + ml)
{
size_t correction = ml - (int)(start2 - ip);
start2 += correction;
ref2 += correction;
ml2 -= correction;
if (ml2 < 3) { ml2 = 0; }
if ((ml2 < minMatchLongOff) && ((U32)(start2 - ref2) >= LZ5_MAX_16BIT_OFFSET)) { ml2 = 0; }
}
_Encode:
if (LZ5_encodeSequence_LZ5v2(ctx, &ip, &anchor, ml, ref)) goto _output_error;
if (ml2)
{
ip = start2; ref = ref2; ml = ml2;
ml2 = 0;
goto _Search;
}
}
/* Encode Last Literals */
ip = iend;
if (LZ5_encodeLastLiterals_LZ5v2(ctx, &ip, &anchor)) goto _output_error;
/* End */
return 1;
_output_error:
return 0;
}
C/lizard/lz5frame.c
+1363
View File
File diff suppressed because it is too large Load Diff
C/lizard/lz5frame.h
+301
View File
@@ -0,0 +1,301 @@
/*
LZ5 auto-framing library
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
/* LZ5F is a stand-alone API to create LZ5-compressed frames
* conformant with specification v1.5.1.
* All related operations, including memory management, are handled internally by the library.
* You don't need lz5_compress.h when using lz5frame.h.
* */
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*-************************************
* Includes
**************************************/
#include <stddef.h> /* size_t */
/*-************************************
* Error management
**************************************/
typedef size_t LZ5F_errorCode_t;
unsigned LZ5F_isError(LZ5F_errorCode_t code);
const char* LZ5F_getErrorName(LZ5F_errorCode_t code); /* return error code string; useful for debugging */
/*-************************************
* Frame compression types
**************************************/
//#define LZ5F_DISABLE_OBSOLETE_ENUMS
#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS
# define LZ5F_OBSOLETE_ENUM(x) ,x
#else
# define LZ5F_OBSOLETE_ENUM(x)
#endif
typedef enum {
LZ5F_default=0,
LZ5F_max128KB=1,
LZ5F_max256KB=2,
LZ5F_max1MB=3,
LZ5F_max4MB=4,
LZ5F_max16MB=5,
LZ5F_max64MB=6,
LZ5F_max256MB=7
} LZ5F_blockSizeID_t;
typedef enum {
LZ5F_blockLinked=0,
LZ5F_blockIndependent
LZ5F_OBSOLETE_ENUM(blockLinked = LZ5F_blockLinked)
LZ5F_OBSOLETE_ENUM(blockIndependent = LZ5F_blockIndependent)
} LZ5F_blockMode_t;
typedef enum {
LZ5F_noContentChecksum=0,
LZ5F_contentChecksumEnabled
LZ5F_OBSOLETE_ENUM(noContentChecksum = LZ5F_noContentChecksum)
LZ5F_OBSOLETE_ENUM(contentChecksumEnabled = LZ5F_contentChecksumEnabled)
} LZ5F_contentChecksum_t;
typedef enum {
LZ5F_frame=0,
LZ5F_skippableFrame
LZ5F_OBSOLETE_ENUM(skippableFrame = LZ5F_skippableFrame)
} LZ5F_frameType_t;
#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS
typedef LZ5F_blockSizeID_t blockSizeID_t;
typedef LZ5F_blockMode_t blockMode_t;
typedef LZ5F_frameType_t frameType_t;
typedef LZ5F_contentChecksum_t contentChecksum_t;
#endif
typedef struct {
LZ5F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ5F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LZ5F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LZ5F_frameType_t frameType; /* LZ5F_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
} LZ5F_frameInfo_t;
typedef struct {
LZ5F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ5F_preferences_t;
/*-*********************************
* Simple compression function
***********************************/
size_t LZ5F_compressFrameBound(size_t srcSize, const LZ5F_preferences_t* preferencesPtr);
/*!LZ5F_compressFrame() :
* Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.1
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ5F_compressFrameBound()
* If this condition is not respected, LZ5F_compressFrame() will fail (result is an errorCode)
* The LZ5F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* The result of the function is the number of bytes written into dstBuffer.
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
*/
size_t LZ5F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_preferences_t* preferencesPtr);
/*-***********************************
* Advanced compression functions
*************************************/
typedef struct LZ5F_cctx_s* LZ5F_compressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ5F_compress(); avoid saving src content within tmp buffer as future dictionary */
unsigned reserved[3];
} LZ5F_compressOptions_t;
/* Resource Management */
#define LZ5F_VERSION 100
LZ5F_errorCode_t LZ5F_createCompressionContext(LZ5F_compressionContext_t* cctxPtr, unsigned version);
LZ5F_errorCode_t LZ5F_freeCompressionContext(LZ5F_compressionContext_t cctx);
/* LZ5F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ5F_createCompressionContext(), which takes as argument a version and an LZ5F_preferences_t structure.
* The version provided MUST be LZ5F_VERSION. It is intended to track potential version differences between different binaries.
* The function will provide a pointer to a fully allocated LZ5F_compressionContext_t object.
* If the result LZ5F_errorCode_t is not zero, there was an error during context creation.
* Object can release its memory using LZ5F_freeCompressionContext();
*/
/* Compression */
size_t LZ5F_compressBegin(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_preferences_t* prefsPtr);
/* LZ5F_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
* The LZ5F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
* The result of the function is the number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ5F_isError())
*/
size_t LZ5F_compressBound(size_t srcSize, const LZ5F_preferences_t* prefsPtr);
/* LZ5F_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
* Different preferences can produce different results.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
*/
size_t LZ5F_compressUpdate(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_compressOptions_t* cOptPtr);
/* LZ5F_compressUpdate()
* LZ5F_compressUpdate() can be called repetitively to compress as much data as necessary.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ5F_compressBound().
* If this condition is not respected, LZ5F_compress() will fail (result is an errorCode).
* LZ5F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
*/
size_t LZ5F_flush(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr);
/* LZ5F_flush()
* Should you need to generate compressed data immediately, without waiting for the current block to be filled,
* you can call LZ5_flush(), which will immediately compress any remaining data buffered within cctx.
* Note that dstMaxSize must be large enough to ensure the operation will be successful.
* LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer
* (it can be zero, this means there was no data left within cctx)
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
*/
size_t LZ5F_compressEnd(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr);
/* LZ5F_compressEnd()
* When you want to properly finish the compressed frame, just call LZ5F_compressEnd().
* It will flush whatever data remained within compressionContext (like LZ5_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
* A successful call to LZ5F_compressEnd() makes cctx available again for next compression task.
*/
/*-*********************************
* Decompression functions
***********************************/
typedef struct LZ5F_dctx_s* LZ5F_decompressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
unsigned reserved[3];
} LZ5F_decompressOptions_t;
/* Resource management */
/*!LZ5F_createDecompressionContext() :
* Create an LZ5F_decompressionContext_t object, which will be used to track all decompression operations.
* The version provided MUST be LZ5F_VERSION. It is intended to track potential breaking differences between different versions.
* The function will provide a pointer to a fully allocated and initialized LZ5F_decompressionContext_t object.
* The result is an errorCode, which can be tested using LZ5F_isError().
* dctx memory can be released using LZ5F_freeDecompressionContext();
* The result of LZ5F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
LZ5F_errorCode_t LZ5F_createDecompressionContext(LZ5F_decompressionContext_t* dctxPtr, unsigned version);
LZ5F_errorCode_t LZ5F_freeDecompressionContext(LZ5F_decompressionContext_t dctx);
/*====== Decompression ======*/
/*!LZ5F_getFrameInfo() :
* This function decodes frame header information (such as max blockSize, frame checksum, etc.).
* Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
* A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that.
* The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
* (note that LZ5F_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough)
* Frame header info is *copied into* an already allocated LZ5F_frameInfo_t structure.
* The function result is an hint about how many srcSize bytes LZ5F_decompress() expects for next call,
* or an error code which can be tested using LZ5F_isError()
* (typically, when there is not enough src bytes to fully decode the frame header)
* Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr)
*/
size_t LZ5F_getFrameInfo(LZ5F_decompressionContext_t dctx,
LZ5F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/*!LZ5F_decompress() :
* Call this function repetitively to regenerate data compressed within srcBuffer.
* The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* If number of bytes read is < number of bytes provided, then decompression operation is not completed.
* It typically happens when dstBuffer is not large enough to contain all decoded data.
* LZ5F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr)
* The function will check this condition, and refuse to continue if it is not respected.
*
* `dstBuffer` is expected to be flushed between each call to the function, its content will be overwritten.
* `dst` arguments can be changed at will at each consecutive call to the function.
*
* The function result is an hint of how many `srcSize` bytes LZ5F_decompress() expects for next call.
* Schematically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
* This is just a hint though, it's always possible to provide any srcSize.
* When a frame is fully decoded, the function result will be 0 (no more data expected).
* If decompression failed, function result is an error code, which can be tested using LZ5F_isError().
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
size_t LZ5F_decompress(LZ5F_decompressionContext_t dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ5F_decompressOptions_t* dOptPtr);
#if defined (__cplusplus)
}
#endif
C/lizard/lz5frame_static.h
+80
View File
@@ -0,0 +1,80 @@
/*
LZ5 auto-framing library
Header File for static linking only
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* lz5frame_static.h should be used solely in the context of static linking.
* It contains definitions which may still change overtime.
* Never use it in the context of DLL linking.
* */
/**************************************
* Includes
**************************************/
#include "lz5frame.h"
/**************************************
* Error management
* ************************************/
#define LZ5F_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_maxCode)
//#define LZ5F_DISABLE_OLD_ENUMS
#ifndef LZ5F_DISABLE_OLD_ENUMS
#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM, ENUM = LZ5F_##ENUM,
#else
#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM,
#endif
typedef enum { LZ5F_LIST_ERRORS(LZ5F_GENERATE_ENUM) } LZ5F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
#if defined (__cplusplus)
}
#endif
C/lizard/mem.h
+394
View File
@@ -0,0 +1,394 @@
/* ******************************************************************
mem.h
low-level memory access routines
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
****************************************************************** */
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */
/*-****************************************
* Compiler specifics
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined(__GNUC__)
# define MEM_STATIC static __inline __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
# define MEM_STATIC static __inline
#else
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
/*-**************************************************************
* Basic Types
*****************************************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef int16_t S16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef signed short S16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
#endif
/*-**************************************************************
* Memory I/O
*****************************************************************/
/* MEM_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets depending on alignment.
* In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
#endif
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
MEM_STATIC unsigned MEM_isLittleEndian(void)
{
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
}
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
/* violates C standard, by lying on structure alignment.
Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
__pragma( pack(push, 1) )
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
__pragma( pack(pop) )
#else
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
#endif
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
/* default method, safe and standard.
can sometimes prove slower */
MEM_STATIC U16 MEM_read16(const void* memPtr)
{
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U32 MEM_read32(const void* memPtr)
{
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U64 MEM_read64(const void* memPtr)
{
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC size_t MEM_readST(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif /* MEM_FORCE_MEMORY_ACCESS */
MEM_STATIC U32 MEM_swap32(U32 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_ulong(in);
#elif defined (__GNUC__)
return __builtin_bswap32(in);
#else
return ((in << 24) & 0xff000000 ) |
((in << 8) & 0x00ff0000 ) |
((in >> 8) & 0x0000ff00 ) |
((in >> 24) & 0x000000ff );
#endif
}
MEM_STATIC U64 MEM_swap64(U64 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_uint64(in);
#elif defined (__GNUC__)
return __builtin_bswap64(in);
#else
return ((in << 56) & 0xff00000000000000ULL) |
((in << 40) & 0x00ff000000000000ULL) |
((in << 24) & 0x0000ff0000000000ULL) |
((in << 8) & 0x000000ff00000000ULL) |
((in >> 8) & 0x00000000ff000000ULL) |
((in >> 24) & 0x0000000000ff0000ULL) |
((in >> 40) & 0x000000000000ff00ULL) |
((in >> 56) & 0x00000000000000ffULL);
#endif
}
MEM_STATIC size_t MEM_swapST(size_t in)
{
if (MEM_32bits())
return (size_t)MEM_swap32((U32)in);
else
return (size_t)MEM_swap64((U64)in);
}
/*=== Little endian r/w ===*/
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read16(memPtr);
else {
const BYTE* p = (const BYTE*)memPtr;
return (U16)(p[0] + (p[1]<<8));
}
}
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
{
if (MEM_isLittleEndian()) {
MEM_write16(memPtr, val);
} else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val;
p[1] = (BYTE)(val>>8);
}
}
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
{
MEM_writeLE16(memPtr, (U16)val);
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
}
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read32(memPtr);
else
return MEM_swap32(MEM_read32(memPtr));
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, val32);
else
MEM_write32(memPtr, MEM_swap32(val32));
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read64(memPtr);
else
return MEM_swap64(MEM_read64(memPtr));
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, val64);
else
MEM_write64(memPtr, MEM_swap64(val64));
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readLE32(memPtr);
else
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
/*=== Big endian r/w ===*/
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap32(MEM_read32(memPtr));
else
return MEM_read32(memPtr);
}
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, MEM_swap32(val32));
else
MEM_write32(memPtr, val32);
}
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap64(MEM_read64(memPtr));
else
return MEM_read64(memPtr);
}
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, MEM_swap64(val64));
else
MEM_write64(memPtr, val64);
}
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readBE32(memPtr);
else
return (size_t)MEM_readBE64(memPtr);
}
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeBE32(memPtr, (U32)val);
else
MEM_writeBE64(memPtr, (U64)val);
}
/* function safe only for comparisons */
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */
C/lizard/xxhash.c
+868
View File
@@ -0,0 +1,868 @@
/*
* xxHash - Fast Hash algorithm
* Copyright (C) 2012-2016, Yann Collet
*
* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* You can contact the author at :
* - xxHash homepage: http://www.xxhash.com
* - xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* *************************************
* Tuning parameters
***************************************/
/*!XXH_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method doesn't depend on compiler but violate C standard.
* It can generate buggy code on targets which do not support unaligned memory accesses.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See https://stackoverflow.com/a/32095106/646947 for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define XXH_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define XXH_FORCE_MEMORY_ACCESS 1
# endif
#endif
/*!XXH_ACCEPT_NULL_INPUT_POINTER :
* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
* By default, this option is disabled. To enable it, uncomment below define :
*/
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
/*!XXH_FORCE_NATIVE_FORMAT :
* By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
* Results are therefore identical for little-endian and big-endian CPU.
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
* Should endian-independance be of no importance for your application, you may set the #define below to 1,
* to improve speed for Big-endian CPU.
* This option has no impact on Little_Endian CPU.
*/
#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */
# define XXH_FORCE_NATIVE_FORMAT 0
#endif
/*!XXH_FORCE_ALIGN_CHECK :
* This is a minor performance trick, only useful with lots of very small keys.
* It means : check for aligned/unaligned input.
* The check costs one initial branch per hash; set to 0 when the input data
* is guaranteed to be aligned.
*/
#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
# define XXH_FORCE_ALIGN_CHECK 0
# else
# define XXH_FORCE_ALIGN_CHECK 1
# endif
#endif
/* *************************************
* Includes & Memory related functions
***************************************/
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
#include <stdlib.h>
static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
/* for memcpy() */
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
#define XXH_STATIC_LINKING_ONLY
#include "xxhash.h"
/* *************************************
* Compiler Specific Options
***************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# define FORCE_INLINE static __forceinline
#else
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/* *************************************
* Basic Types
***************************************/
#ifndef MEM_MODULE
# define MEM_MODULE
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
# else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
# endif
#endif
#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; }
static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; }
#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign;
static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
#else
/* portable and safe solution. Generally efficient.
* see : https://stackoverflow.com/a/32095106/646947
*/
static U32 XXH_read32(const void* memPtr)
{
U32 val;
memcpy(&val, memPtr, sizeof(val));
return val;
}
static U64 XXH_read64(const void* memPtr)
{
U64 val;
memcpy(&val, memPtr, sizeof(val));
return val;
}
#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
/* ****************************************
* Compiler-specific Functions and Macros
******************************************/
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
#if defined(_MSC_VER)
# define XXH_rotl32(x,r) _rotl(x,r)
# define XXH_rotl64(x,r) _rotl64(x,r)
#else
# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
#endif
#if defined(_MSC_VER) /* Visual Studio */
# define XXH_swap32 _byteswap_ulong
# define XXH_swap64 _byteswap_uint64
#elif GCC_VERSION >= 403
# define XXH_swap32 __builtin_bswap32
# define XXH_swap64 __builtin_bswap64
#else
static U32 XXH_swap32 (U32 x)
{
return ((x << 24) & 0xff000000 ) |
((x << 8) & 0x00ff0000 ) |
((x >> 8) & 0x0000ff00 ) |
((x >> 24) & 0x000000ff );
}
static U64 XXH_swap64 (U64 x)
{
return ((x << 56) & 0xff00000000000000ULL) |
((x << 40) & 0x00ff000000000000ULL) |
((x << 24) & 0x0000ff0000000000ULL) |
((x << 8) & 0x000000ff00000000ULL) |
((x >> 8) & 0x00000000ff000000ULL) |
((x >> 24) & 0x0000000000ff0000ULL) |
((x >> 40) & 0x000000000000ff00ULL) |
((x >> 56) & 0x00000000000000ffULL);
}
#endif
/* *************************************
* Architecture Macros
***************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */
#ifndef XXH_CPU_LITTLE_ENDIAN
static const int g_one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one))
#endif
/* ***************************
* Memory reads
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
else
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
static U32 XXH_readBE32(const void* ptr)
{
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
}
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
else
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
}
FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
static U64 XXH_readBE64(const void* ptr)
{
return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
}
/* *************************************
* Macros
***************************************/
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/* *************************************
* Constants
***************************************/
static const U32 PRIME32_1 = 2654435761U;
static const U32 PRIME32_2 = 2246822519U;
static const U32 PRIME32_3 = 3266489917U;
static const U32 PRIME32_4 = 668265263U;
static const U32 PRIME32_5 = 374761393U;
static const U64 PRIME64_1 = 11400714785074694791ULL;
static const U64 PRIME64_2 = 14029467366897019727ULL;
static const U64 PRIME64_3 = 1609587929392839161ULL;
static const U64 PRIME64_4 = 9650029242287828579ULL;
static const U64 PRIME64_5 = 2870177450012600261ULL;
XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
/* **************************
* Utils
****************************/
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState)
{
memcpy(dstState, srcState, sizeof(*dstState));
}
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState)
{
memcpy(dstState, srcState, sizeof(*dstState));
}
/* ***************************
* Simple Hash Functions
*****************************/
static U32 XXH32_round(U32 seed, U32 input)
{
seed += input * PRIME32_2;
seed = XXH_rotl32(seed, 13);
seed *= PRIME32_1;
return seed;
}
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
U32 h32;
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)16;
}
#endif
if (len>=16) {
const BYTE* const limit = bEnd - 16;
U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0;
U32 v4 = seed - PRIME32_1;
do {
v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4;
v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4;
v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4;
v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4;
} while (p<=limit);
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
} else {
h32 = seed + PRIME32_5;
}
h32 += (U32) len;
while (p+4<=bEnd) {
h32 += XXH_get32bits(p) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
p+=4;
}
while (p<bEnd) {
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
p++;
}
h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
h32 *= PRIME32_3;
h32 ^= h32 >> 16;
return h32;
}
XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH32_CREATESTATE_STATIC(state);
XXH32_reset(state, seed);
XXH32_update(state, input, len);
return XXH32_digest(state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if (XXH_FORCE_ALIGN_CHECK) {
if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
} }
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
static U64 XXH64_round(U64 acc, U64 input)
{
acc += input * PRIME64_2;
acc = XXH_rotl64(acc, 31);
acc *= PRIME64_1;
return acc;
}
static U64 XXH64_mergeRound(U64 acc, U64 val)
{
val = XXH64_round(0, val);
acc ^= val;
acc = acc * PRIME64_1 + PRIME64_4;
return acc;
}
FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
U64 h64;
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL) {
len=0;
bEnd=p=(const BYTE*)(size_t)32;
}
#endif
if (len>=32) {
const BYTE* const limit = bEnd - 32;
U64 v1 = seed + PRIME64_1 + PRIME64_2;
U64 v2 = seed + PRIME64_2;
U64 v3 = seed + 0;
U64 v4 = seed - PRIME64_1;
do {
v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8;
v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8;
v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8;
v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8;
} while (p<=limit);
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
h64 = XXH64_mergeRound(h64, v1);
h64 = XXH64_mergeRound(h64, v2);
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
} else {
h64 = seed + PRIME64_5;
}
h64 += (U64) len;
while (p+8<=bEnd) {
U64 const k1 = XXH64_round(0, XXH_get64bits(p));
h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
if (p+4<=bEnd) {
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
while (p<bEnd) {
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH64_CREATESTATE_STATIC(state);
XXH64_reset(state, seed);
XXH64_update(state, input, len);
return XXH64_digest(state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if (XXH_FORCE_ALIGN_CHECK) {
if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
} }
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
/* **************************************************
* Advanced Hash Functions
****************************************************/
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
{
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
{
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
/*** Hash feed ***/
XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed)
{
XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
memset(&state, 0, sizeof(state));
state.seed = seed;
state.v1 = seed + PRIME32_1 + PRIME32_2;
state.v2 = seed + PRIME32_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME32_1;
memcpy(statePtr, &state, sizeof(state));
return XXH_OK;
}
XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed)
{
XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */
memset(&state, 0, sizeof(state));
state.seed = seed;
state.v1 = seed + PRIME64_1 + PRIME64_2;
state.v2 = seed + PRIME64_2;
state.v3 = seed + 0;
state.v4 = seed - PRIME64_1;
memcpy(statePtr, &state, sizeof(state));
return XXH_OK;
}
FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
if (state->memsize + len < 16) { /* fill in tmp buffer */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
if (state->memsize) { /* some data left from previous update */
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
{ const U32* p32 = state->mem32;
state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++;
state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++;
state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++;
state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++;
}
p += 16-state->memsize;
state->memsize = 0;
}
if (p <= bEnd-16) {
const BYTE* const limit = bEnd - 16;
U32 v1 = state->v1;
U32 v2 = state->v2;
U32 v3 = state->v3;
U32 v4 = state->v4;
do {
v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4;
v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4;
v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4;
v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4;
} while (p<=limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd) {
XXH_memcpy(state->mem32, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
return XXH_OK;
}
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
else
return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian)
{
const BYTE * p = (const BYTE*)state->mem32;
const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
if (state->total_len >= 16) {
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
} else {
h32 = state->seed + PRIME32_5;
}
h32 += (U32) state->total_len;
while (p+4<=bEnd) {
h32 += XXH_readLE32(p, endian) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
p+=4;
}
while (p<bEnd) {
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1;
p++;
}
h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
h32 *= PRIME32_3;
h32 ^= h32 >> 16;
return h32;
}
XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_digest_endian(state_in, XXH_littleEndian);
else
return XXH32_digest_endian(state_in, XXH_bigEndian);
}
/* **** XXH64 **** */
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian)
{
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
if (state->memsize + len < 32) { /* fill in tmp buffer */
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
if (state->memsize) { /* tmp buffer is full */
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian));
state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian));
state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian));
state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian));
p += 32-state->memsize;
state->memsize = 0;
}
if (p+32 <= bEnd) {
const BYTE* const limit = bEnd - 32;
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
do {
v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8;
v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8;
v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8;
v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8;
} while (p<=limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd) {
XXH_memcpy(state->mem64, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
return XXH_OK;
}
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
else
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian)
{
const BYTE * p = (const BYTE*)state->mem64;
const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
if (state->total_len >= 32) {
U64 const v1 = state->v1;
U64 const v2 = state->v2;
U64 const v3 = state->v3;
U64 const v4 = state->v4;
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
h64 = XXH64_mergeRound(h64, v1);
h64 = XXH64_mergeRound(h64, v2);
h64 = XXH64_mergeRound(h64, v3);
h64 = XXH64_mergeRound(h64, v4);
} else {
h64 = state->seed + PRIME64_5;
}
h64 += (U64) state->total_len;
while (p+8<=bEnd) {
U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian));
h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
if (p+4<=bEnd) {
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
while (p<bEnd) {
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_digest_endian(state_in, XXH_littleEndian);
else
return XXH64_digest_endian(state_in, XXH_bigEndian);
}
/* **************************
* Canonical representation
****************************/
/*! Default XXH result types are basic unsigned 32 and 64 bits.
* The canonical representation follows human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs.
*/
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
{
XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
memcpy(dst, &hash, sizeof(*dst));
}
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash)
{
XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
memcpy(dst, &hash, sizeof(*dst));
}
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
{
return XXH_readBE32(src);
}
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
{
return XXH_readBE64(src);
}
C/lizard/xxhash.h
+297
View File
@@ -0,0 +1,297 @@
/*
xxHash - Extremely Fast Hash algorithm
Header File
Copyright (C) 2012-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* Notice extracted from xxHash homepage :
xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
It also successfully passes all tests from the SMHasher suite.
Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
Name Speed Q.Score Author
xxHash 5.4 GB/s 10
CrapWow 3.2 GB/s 2 Andrew
MumurHash 3a 2.7 GB/s 10 Austin Appleby
SpookyHash 2.0 GB/s 10 Bob Jenkins
SBox 1.4 GB/s 9 Bret Mulvey
Lookup3 1.2 GB/s 9 Bob Jenkins
SuperFastHash 1.2 GB/s 1 Paul Hsieh
CityHash64 1.05 GB/s 10 Pike & Alakuijala
FNV 0.55 GB/s 5 Fowler, Noll, Vo
CRC32 0.43 GB/s 9
MD5-32 0.33 GB/s 10 Ronald L. Rivest
SHA1-32 0.28 GB/s 10
Q.Score is a measure of quality of the hash function.
It depends on successfully passing SMHasher test set.
10 is a perfect score.
A 64-bits version, named XXH64, is available since r35.
It offers much better speed, but for 64-bits applications only.
Name Speed on 64 bits Speed on 32 bits
XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s
*/
#ifndef XXHASH_H_5627135585666179
#define XXHASH_H_5627135585666179 1
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************
* Definitions
******************************/
#include <stddef.h> /* size_t */
typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
/* ****************************
* API modifier
******************************/
/** XXH_PRIVATE_API
* This is useful if you want to include xxhash functions in `static` mode
* in order to inline them, and remove their symbol from the public list.
* Methodology :
* #define XXH_PRIVATE_API
* #include "xxhash.h"
* `xxhash.c` is automatically included, so the file is still needed,
* but it's not useful to compile and link it anymore.
*/
#ifdef XXH_PRIVATE_API
# ifndef XXH_STATIC_LINKING_ONLY
# define XXH_STATIC_LINKING_ONLY
# endif
# if defined(__GNUC__)
# define XXH_PUBLIC_API static __attribute__((unused))
# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define XXH_PUBLIC_API static inline
# elif defined(_MSC_VER)
# define XXH_PUBLIC_API static __inline
# else
# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */
# endif
#else
# define XXH_PUBLIC_API /* do nothing */
#endif /* XXH_PRIVATE_API */
/*!XXH_NAMESPACE, aka Namespace Emulation :
If you want to include _and expose_ xxHash functions from within your own library,
but also want to avoid symbol collisions with another library which also includes xxHash,
you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library
with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values).
Note that no change is required within the calling program as long as it includes `xxhash.h` :
regular symbol name will be automatically translated by this header.
*/
#ifdef XXH_NAMESPACE
# define XXH_CAT(A,B) A##B
# define XXH_NAME2(A,B) XXH_CAT(A,B)
# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
#endif
/* *************************************
* Version
***************************************/
#define XXH_VERSION_MAJOR 0
#define XXH_VERSION_MINOR 6
#define XXH_VERSION_RELEASE 1
#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
XXH_PUBLIC_API unsigned XXH_versionNumber (void);
/* ****************************
* Simple Hash Functions
******************************/
typedef unsigned int XXH32_hash_t;
typedef unsigned long long XXH64_hash_t;
XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed);
XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed);
/*!
XXH32() :
Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
The memory between input & input+length must be valid (allocated and read-accessible).
"seed" can be used to alter the result predictably.
Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
XXH64() :
Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
"seed" can be used to alter the result predictably.
This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark).
*/
/* ****************************
* Streaming Hash Functions
******************************/
typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */
typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
/*! State allocation, compatible with dynamic libraries */
XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void);
XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
/* hash streaming */
XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed);
XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr);
/*
These functions generate the xxHash of an input provided in multiple segments.
Note that, for small input, they are slower than single-call functions, due to state management.
For small input, prefer `XXH32()` and `XXH64()` .
XXH state must first be allocated, using XXH*_createState() .
Start a new hash by initializing state with a seed, using XXH*_reset().
Then, feed the hash state by calling XXH*_update() as many times as necessary.
Obviously, input must be allocated and read accessible.
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, a hash value can be produced anytime, by using XXH*_digest().
This function returns the nn-bits hash as an int or long long.
It's still possible to continue inserting input into the hash state after a digest,
and generate some new hashes later on, by calling again XXH*_digest().
When done, free XXH state space if it was allocated dynamically.
*/
/* **************************
* Utils
****************************/
#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */
# define restrict /* disable restrict */
#endif
XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state);
XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state);
/* **************************
* Canonical representation
****************************/
typedef struct { unsigned char digest[4]; } XXH32_canonical_t;
typedef struct { unsigned char digest[8]; } XXH64_canonical_t;
XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash);
XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src);
/* Default result type for XXH functions are primitive unsigned 32 and 64 bits.
* The canonical representation uses human-readable write convention, aka big-endian (large digits first).
* These functions allow transformation of hash result into and from its canonical format.
* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs.
*/
#ifdef XXH_STATIC_LINKING_ONLY
/* ================================================================================================
This section contains definitions which are not guaranteed to remain stable.
They could change in a future version, becoming incompatible with a different version of the library.
They shall only be used with static linking.
=================================================================================================== */
/* These definitions allow allocating XXH state statically (on stack) */
struct XXH32_state_s {
unsigned long long total_len;
unsigned seed;
unsigned v1;
unsigned v2;
unsigned v3;
unsigned v4;
unsigned mem32[4]; /* buffer defined as U32 for alignment */
unsigned memsize;
}; /* typedef'd to XXH32_state_t */
struct XXH64_state_s {
unsigned long long total_len;
unsigned long long seed;
unsigned long long v1;
unsigned long long v2;
unsigned long long v3;
unsigned long long v4;
unsigned long long mem64[4]; /* buffer defined as U64 for alignment */
unsigned memsize;
}; /* typedef'd to XXH64_state_t */
# ifdef XXH_PRIVATE_API
# include "xxhash.c" /* include xxhash functions as `static`, for inlining */
# endif
#endif /* XXH_STATIC_LINKING_ONLY */
#if defined (__cplusplus)
}
#endif
#endif /* XXHASH_H_5627135585666179 */