diff --git a/C/7zVersion.h b/C/7zVersion.h index 97d55721..893d4e38 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -3,12 +3,12 @@ #define MY_VER_BUILD 0 #define MY_VERSION_NUMBERS "16.04 ZS" #define MY_VERSION "16.04 ZS" -#define MY_DATE "2016-10-06" +#define MY_DATE "2016-10-16" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "(c) 1999-2016 Igor Pavlov, Tino Reichardt" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2016 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h index 727efda3..9f831783 100644 --- a/C/7zVersionTr.h +++ b/C/7zVersionTr.h @@ -3,7 +3,7 @@ #define MY_VER_BUILD 0 #define MY_VERSION_NUMBERS "1.1.0" #define MY_VERSION "1.1.0" -#define MY_DATE "2016-10-06" +#define MY_DATE "2016-10-16" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Tino Reichardt" diff --git a/C/lz4/lz4.c b/C/lz4/lz4.c new file mode 100644 index 00000000..08cf6b5c --- /dev/null +++ b/C/lz4/lz4.c @@ -0,0 +1,1516 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + +/************************************** +* Tuning parameters +**************************************/ +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#define HEAPMODE 0 + +/* + * ACCELERATION_DEFAULT : + * Select "acceleration" for LZ4_compress_fast() when parameter value <= 0 + */ +#define ACCELERATION_DEFAULT 1 + + +/************************************** +* CPU Feature Detection +**************************************/ +/* + * LZ4_FORCE_SW_BITCOUNT + * Define this parameter if your target system or compiler does not support hardware bit count + */ +#if defined(_MSC_VER) && defined(_WIN32_WCE) /* Visual Studio for Windows CE does not support Hardware bit count */ +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +/************************************** +* Includes +**************************************/ +#include "lz4.h" + + +/************************************** +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* _MSC_VER */ + +/* LZ4_GCC_VERSION is defined into lz4.h */ +#if (LZ4_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +/************************************** +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + + +/************************************** +* Basic Types +**************************************/ +#if defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + + +/************************************** +* Reading and writing into memory +**************************************/ +#define STEPSIZE sizeof(size_t) + +static unsigned LZ4_64bits(void) { return sizeof(void*)==8; } + +static unsigned LZ4_isLittleEndian(void) +{ + const union { U32 i; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + + +static U16 LZ4_read16(const void* memPtr) +{ + U16 val16; + memcpy(&val16, memPtr, 2); + return val16; +} + +static U16 LZ4_readLE16(const void* memPtr) +{ + if (LZ4_isLittleEndian()) + { + return LZ4_read16(memPtr); + } + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U16)((U16)p[0] + (p[1]<<8)); + } +} + +static void LZ4_writeLE16(void* memPtr, U16 value) +{ + if (LZ4_isLittleEndian()) + { + memcpy(memPtr, &value, 2); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + } +} + +static U32 LZ4_read32(const void* memPtr) +{ + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} + +static U64 LZ4_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} + +static size_t LZ4_read_ARCH(const void* p) +{ + if (LZ4_64bits()) + return (size_t)LZ4_read64(p); + else + return (size_t)LZ4_read32(p); +} + + +static void LZ4_copy4(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 4); } + +static void LZ4_copy8(void* dstPtr, const void* srcPtr) { memcpy(dstPtr, srcPtr, 8); } + +/* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ +static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) +{ + BYTE* d = (BYTE*)dstPtr; + const BYTE* s = (const BYTE*)srcPtr; + BYTE* e = (BYTE*)dstEnd; + do { LZ4_copy8(d,s); d+=8; s+=8; } while (d>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } + else /* Big Endian CPU */ + { + if (LZ4_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll((U64)val) >> 3); +# else + unsigned r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif (defined(__clang__) || (LZ4_GCC_VERSION >= 304)) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + +static unsigned LZ4_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while (likely(pIn compression run slower on incompressible data */ + + +/************************************** +* Local Structures and types +**************************************/ +typedef struct { + U32 hashTable[HASH_SIZE_U32]; + U32 currentOffset; + U32 initCheck; + const BYTE* dictionary; + BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ + U32 dictSize; +} LZ4_stream_t_internal; + +typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; +typedef enum { byPtr, byU32, byU16 } tableType_t; + +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + + +/************************************** +* Local Utils +**************************************/ +int LZ4_versionNumber (void) { return LZ4_VERSION_NUMBER; } +int LZ4_compressBound(int isize) { return LZ4_COMPRESSBOUND(isize); } +int LZ4_sizeofState() { return LZ4_STREAMSIZE; } + + + +/******************************** +* Compression functions +********************************/ + +static U32 LZ4_hashSequence(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-(LZ4_HASHLOG+1))); + else + return (((sequence) * 2654435761U) >> ((MINMATCH*8)-LZ4_HASHLOG)); +} + +static const U64 prime5bytes = 889523592379ULL; +static U32 LZ4_hashSequence64(size_t sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ4_HASHLOG+1 : LZ4_HASHLOG; + const U32 hashMask = (1<> (40 - hashLog)) & hashMask; +} + +static U32 LZ4_hashSequenceT(size_t sequence, tableType_t const tableType) +{ + if (LZ4_64bits()) + return LZ4_hashSequence64(sequence, tableType); + return LZ4_hashSequence((U32)sequence, tableType); +} + +static U32 LZ4_hashPosition(const void* p, tableType_t tableType) { return LZ4_hashSequenceT(LZ4_read_ARCH(p), tableType); } + +static void LZ4_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +static void LZ4_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + LZ4_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +static const BYTE* LZ4_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +static const BYTE* LZ4_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ4_hashPosition(p, tableType); + return LZ4_getPositionOnHash(h, tableBase, tableType, srcBase); +} + +FORCE_INLINE int LZ4_compress_generic( + void* const ctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) +{ + LZ4_stream_t_internal* const dictPtr = (LZ4_stream_t_internal*)ctx; + + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - dictPtr->dictSize; + const BYTE* const dictionary = dictPtr->dictionary; + const BYTE* const dictEnd = dictionary + dictPtr->dictSize; + const size_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH; + size_t refDelta=0; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source - dictPtr->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSize> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx, tableType, base); + if (dict==usingExtDict) + { + if (match<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match+refDelta) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; /* Check output limit */ + if (litLength>=RUN_MASK) + { + int len = (int)litLength-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< matchlimit) limit = matchlimit; + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchLength; + if (ip==limit) + { + unsigned more = LZ4_count(ip, (const BYTE*)source, matchlimit); + matchLength += more; + ip += more; + } + } + else + { + matchLength = LZ4_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchLength; + } + + if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) + return 0; /* Check output limit */ + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } + if (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx, tableType, base); + if (dict==usingExtDict) + { + if (match<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + LZ4_putPosition(ip, ctx, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match+refDelta)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + const size_t lastRun = (size_t)(iend - anchor); + if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) + return 0; /* Check output limit */ + if (lastRun >= RUN_MASK) + { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRun<= LZ4_compressBound(inputSize)) + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(state, source, dest, inputSize, 0, notLimited, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } + else + { + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } +} + + +int LZ4_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ +#if (HEAPMODE) + void* ctxPtr = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctx; + void* ctxPtr = &ctx; +#endif + + int result = LZ4_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + +#if (HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ4_compress_fast(source, dest, inputSize, maxOutputSize, 1); +} + + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ4_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t ctx; + + LZ4_resetStream(&ctx); + + if (inputSize < LZ4_64Klimit) + return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ4_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, LZ4_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); +} + + +/******************************** +* destSize variant +********************************/ + +static int LZ4_compress_destSize_generic( + void* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ4_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ4_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtr> LZ4_skipTrigger); + + if (unlikely(forwardIp > mflimit)) + goto _last_literals; + + match = LZ4_getPositionOnHash(h, ctx, tableType, base); + forwardH = LZ4_hashPosition(forwardIp, tableType); + LZ4_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (LZ4_read32(match) != LZ4_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) + { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + if (litLength>=RUN_MASK) + { + unsigned len = litLength - RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength< oMaxMatch) + { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + //printf("offset %5i, matchLength%5i \n", (int)(ip-match), matchLength + MINMATCH); + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ4_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ4_getPosition(ip, ctx, tableType, base); + LZ4_putPosition(ip, ctx, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (LZ4_read32(match)==LZ4_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ4_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) + { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) + { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRunSize<= LZ4_compressBound(*srcSizePtr)) /* compression success is guaranteed */ + { + return LZ4_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } + else + { + if (*srcSizePtr < LZ4_64Klimit) + return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ4_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, LZ4_64bits() ? byU32 : byPtr); + } +} + + +int LZ4_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (HEAPMODE) + void* ctx = ALLOCATOR(1, sizeof(LZ4_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ4_stream_t ctxBody; + void* ctx = &ctxBody; +#endif + + int result = LZ4_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/******************************** +* Streaming functions +********************************/ + +LZ4_stream_t* LZ4_createStream(void) +{ + LZ4_stream_t* lz4s = (LZ4_stream_t*)ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_STATIC_ASSERT(LZ4_STREAMSIZE >= sizeof(LZ4_stream_t_internal)); /* A compilation error here means LZ4_STREAMSIZE is not large enough */ + LZ4_resetStream(lz4s); + return lz4s; +} + +void LZ4_resetStream (LZ4_stream_t* LZ4_stream) +{ + MEM_INIT(LZ4_stream, 0, sizeof(LZ4_stream_t)); +} + +int LZ4_freeStream (LZ4_stream_t* LZ4_stream) +{ + FREEMEM(LZ4_stream); + return (0); +} + + +#define HASH_UNIT sizeof(size_t) +int LZ4_loadDict (LZ4_stream_t* LZ4_dict, const char* dictionary, int dictSize) +{ + LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ4_resetStream(LZ4_dict); + + if (dictSize < (int)HASH_UNIT) + { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + } + + if ((dictEnd - p) > 64 KB) p = dictEnd - 64 KB; + dict->currentOffset += 64 KB; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) + { + LZ4_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } + + return dict->dictSize; +} + + +static void LZ4_renormDictT(LZ4_stream_t_internal* LZ4_dict, const BYTE* src) +{ + if ((LZ4_dict->currentOffset > 0x80000000) || + ((size_t)LZ4_dict->currentOffset > (size_t)src)) /* address space overflow */ + { + /* rescale hash table */ + U32 delta = LZ4_dict->currentOffset - 64 KB; + const BYTE* dictEnd = LZ4_dict->dictionary + LZ4_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ4_dict->hashTable[i]=0; + else LZ4_dict->hashTable[i] -= delta; + } + LZ4_dict->currentOffset = 64 KB; + if (LZ4_dict->dictSize > 64 KB) LZ4_dict->dictSize = 64 KB; + LZ4_dict->dictionary = dictEnd - LZ4_dict->dictSize; + } +} + + +int LZ4_compress_fast_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_stream; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ4_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) + { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > 64 KB) streamPtr->dictSize = 64 KB; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) + { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { + int result; + if ((streamPtr->dictSize < 64 KB) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ4_compress_generic(LZ4_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force external dictionary mode */ +int LZ4_compress_forceExtDict (LZ4_stream_t* LZ4_dict, const char* source, char* dest, int inputSize) +{ + LZ4_stream_t_internal* streamPtr = (LZ4_stream_t_internal*)LZ4_dict; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ4_renormDictT((LZ4_stream_t_internal*)LZ4_dict, smallest); + + result = LZ4_compress_generic(LZ4_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + + return result; +} + + +int LZ4_saveDict (LZ4_stream_t* LZ4_dict, char* safeBuffer, int dictSize) +{ + LZ4_stream_t_internal* dict = (LZ4_stream_t_internal*) LZ4_dict; + const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; + + if ((U32)dictSize > 64 KB) dictSize = 64 KB; /* useless to define a dictionary > 64 KB */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/******************************* +* Decompression functions +*******************************/ +/* + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is essential this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int LZ4_decompress_generic( + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const size_t dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; + const size_t dec64table[] = {0, 0, 0, (size_t)-1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(64 KB))); + + + /* Special cases */ + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + + /* Main Loop */ + while (1) + { + unsigned token; + size_t length; + const BYTE* match; + + /* get literal length */ + token = *ip++; + if ((length=(token>>ML_BITS)) == RUN_MASK) + { + unsigned s; + do + { + s = *ip++; + length += s; + } + while (likely((endOnInput)?ip(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(2+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-COPYLENGTH))) + { + if (partialDecoding) + { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } + else + { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + LZ4_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ + match = cpy - LZ4_readLE16(ip); ip+=2; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside destination buffer */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) + { + unsigned s; + do + { + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + s = *ip++; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) + { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) + { + /* match can be copied as a single segment from external dictionary */ + match = dictEnd - (lowPrefix-match); + memmove(op, match, length); op += length; + } + else + { + /* match encompass external dictionary and current segment */ + size_t copySize = (size_t)(lowPrefix-match); + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + copySize = length - copySize; + if (copySize > (size_t)(op-lowPrefix)) /* overlap within current segment */ + { + BYTE* const endOfMatch = op + copySize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } + else + { + memcpy(op, lowPrefix, copySize); + op += copySize; + } + } + continue; + } + + /* copy repeated sequence */ + cpy = op + length; + if (unlikely((op-match)<8)) + { + const size_t dec64 = dec64table[op-match]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[op-match]; + LZ4_copy4(op+4, match); + op += 8; match -= dec64; + } else { LZ4_copy8(op, match); op+=8; match+=8; } + + if (unlikely(cpy>oend-12)) + { + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals */ + if (op < oend-8) + { + LZ4_wildCopy(op, match, oend-8); + match += (oend-8) - op; + op = oend-8; + } + while (opprefixSize = (size_t) dictSize; + lz4sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz4sd->externalDict = NULL; + lz4sd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) + { + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += result; + lz4sd->prefixEnd += result; + } + else + { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = lz4sd->prefixEnd - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = result; + lz4sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ4_streamDecode_t_internal* lz4sd = (LZ4_streamDecode_t_internal*) LZ4_streamDecode; + int result; + + if (lz4sd->prefixEnd == (BYTE*)dest) + { + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz4sd->prefixEnd - lz4sd->prefixSize, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize += originalSize; + lz4sd->prefixEnd += originalSize; + } + else + { + lz4sd->extDictSize = lz4sd->prefixSize; + lz4sd->externalDict = (BYTE*)dest - lz4sd->extDictSize; + result = LZ4_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz4sd->externalDict, lz4sd->extDictSize); + if (result <= 0) return result; + lz4sd->prefixSize = originalSize; + lz4sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +FORCE_INLINE int LZ4_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(64 KB - 1)) + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-64 KB, NULL, 0); + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +int LZ4_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +} + +int LZ4_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); +} + +/* debug function */ +int LZ4_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + + +/*************************************************** +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ4_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ4_compress(const char* source, char* dest, int inputSize) { return LZ4_compress_default(source, dest, inputSize, LZ4_compressBound(inputSize)); } +int LZ4_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ4_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_fast_extState(state, src, dst, srcSize, LZ4_compressBound(srcSize), 1); } +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_fast_continue(LZ4_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ4_compress_continue (LZ4_stream_t* LZ4_stream, const char* source, char* dest, int inputSize) { return LZ4_compress_fast_continue(LZ4_stream, source, dest, inputSize, LZ4_compressBound(inputSize), 1); } + +/* +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ4_uncompress is totally equivalent to LZ4_decompress_fast +- LZ4_uncompress_unknownOutputSize is totally equivalent to LZ4_decompress_safe +*/ +int LZ4_uncompress (const char* source, char* dest, int outputSize) { return LZ4_decompress_fast(source, dest, outputSize); } +int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ4_decompress_safe(source, dest, isize, maxOutputSize); } + + +/* Obsolete Streaming functions */ + +int LZ4_sizeofStreamState() { return LZ4_STREAMSIZE; } + +static void LZ4_init(LZ4_stream_t_internal* lz4ds, BYTE* base) +{ + MEM_INIT(lz4ds, 0, LZ4_STREAMSIZE); + lz4ds->bufferStart = base; +} + +int LZ4_resetStreamState(void* state, char* inputBuffer) +{ + if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ4_init((LZ4_stream_t_internal*)state, (BYTE*)inputBuffer); + return 0; +} + +void* LZ4_create (char* inputBuffer) +{ + void* lz4ds = ALLOCATOR(8, LZ4_STREAMSIZE_U64); + LZ4_init ((LZ4_stream_t_internal*)lz4ds, (BYTE*)inputBuffer); + return lz4ds; +} + +char* LZ4_slideInputBuffer (void* LZ4_Data) +{ + LZ4_stream_t_internal* ctx = (LZ4_stream_t_internal*)LZ4_Data; + int dictSize = LZ4_saveDict((LZ4_stream_t*)LZ4_Data, (char*)ctx->bufferStart, 64 KB); + return (char*)(ctx->bufferStart + dictSize); +} + +/* Obsolete streaming decompression functions */ + +int LZ4_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ4_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +int LZ4_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ4_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - 64 KB, NULL, 64 KB); +} + +#endif /* LZ4_COMMONDEFS_ONLY */ + diff --git a/C/lz4/lz4.h b/C/lz4/lz4.h new file mode 100644 index 00000000..3e740022 --- /dev/null +++ b/C/lz4/lz4.h @@ -0,0 +1,360 @@ +/* + LZ4 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* + * lz4.h provides block compression functions, and gives full buffer control to programmer. + * If you need to generate inter-operable compressed data (respecting LZ4 frame specification), + * and can let the library handle its own memory, please use lz4frame.h instead. +*/ + +/************************************** +* Version +**************************************/ +#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */ +#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ +#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE) +int LZ4_versionNumber (void); + +/************************************** +* Tuning parameter +**************************************/ +/* + * LZ4_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache + */ +#define LZ4_MEMORY_USAGE 14 + + +/************************************** +* Simple Functions +**************************************/ + +int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); +int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + +/* +LZ4_compress_default() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails + +LZ4_decompress_safe() : + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. +*/ + + +/************************************** +* Advanced Functions +**************************************/ +#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16) + +/* +LZ4_compressBound() : + Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize) + inputSize : max supported value is LZ4_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE) +*/ +int LZ4_compressBound(int inputSize); + +/* +LZ4_compress_fast() : + Same as LZ4_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ4_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1. +*/ +int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); + + +/* +LZ4_compress_fast_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use LZ4_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +int LZ4_sizeofState(void); +int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); + + +/* +LZ4_compress_destSize() : + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails +*/ +int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); + + +/* +LZ4_decompress_fast() : + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ4_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). +*/ +int LZ4_decompress_fast (const char* source, char* dest, int originalSize); + +/* +LZ4_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + +/*********************************************** +* Streaming Compression Functions +***********************************************/ +#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4) +#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long)) +/* + * LZ4_stream_t + * information structure to track an LZ4 stream. + * important : init this structure content before first use ! + * note : only allocated directly the structure if you are statically linking LZ4 + * If you are using liblz4 as a DLL, please use below construction methods instead. + */ +typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t; + +/* + * LZ4_resetStream + * Use this function to init an allocated LZ4_stream_t structure + */ +void LZ4_resetStream (LZ4_stream_t* streamPtr); + +/* + * LZ4_createStream will allocate and initialize an LZ4_stream_t structure + * LZ4_freeStream releases its memory. + * In the context of a DLL (liblz4), please use these methods rather than the static struct. + * They are more future proof, in case of a change of LZ4_stream_t size. + */ +LZ4_stream_t* LZ4_createStream(void); +int LZ4_freeStream (LZ4_stream_t* streamPtr); + +/* + * LZ4_loadDict + * Use this function to load a static dictionary into LZ4_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= 64 KB) + */ +int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize); + +/* + * LZ4_compress_fast_continue + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); + +/* + * LZ4_saveDict + * If previously compressed data block is not guaranteed to remain available at its memory location + * save it into a safer place (char* safeBuffer) + * Note : you don't need to call LZ4_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue() + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error + */ +int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize); + + +/************************************************ +* Streaming Decompression Functions +************************************************/ + +#define LZ4_STREAMDECODESIZE_U64 4 +#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t; +/* + * LZ4_streamDecode_t + * information structure to track an LZ4 stream. + * init this structure content using LZ4_setStreamDecode or memset() before first use ! + * + * In the context of a DLL (liblz4) please prefer usage of construction methods below. + * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future. + * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure + * LZ4_freeStreamDecode releases its memory. + */ +LZ4_streamDecode_t* LZ4_createStreamDecode(void); +int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream); + +/* + * LZ4_setStreamDecode + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * Return : 1 if OK, 0 if error + */ +int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize); + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ4_setStreamDecode() +*/ +int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize); + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as + a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue() + They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure. +*/ +int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); + + + +/************************************** +* Obsolete Functions +**************************************/ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ +#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */ + +/* Obsolete compression functions */ +/* These functions are planned to start generate warnings by r131 approximately */ +int LZ4_compress (const char* source, char* dest, int sourceSize); +int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize); +int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete decompression functions */ +/* These function names are completely deprecated and must no longer be used. + They are only provided here for compatibility with older programs. + - LZ4_uncompress is the same as LZ4_decompress_fast + - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe + These function prototypes are now disabled; uncomment them only if you really need them. + It is highly recommended to stop using these prototypes and migrate to maintained ones */ +/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */ +/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ + +/* Obsolete streaming functions; use new streaming interface whenever possible */ +LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer); +LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void); +LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state); + +/* Obsolete streaming decoding functions */ +LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz4/lz4frame.c b/C/lz4/lz4frame.c new file mode 100644 index 00000000..e5458bb9 --- /dev/null +++ b/C/lz4/lz4frame.c @@ -0,0 +1,1479 @@ +/* +LZ4 auto-framing library +Copyright (C) 2011-2015, Yann Collet. + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : https://github.com/Cyan4973/lz4 +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API to create LZ4-compressed Frames +* in full conformance with specification v1.5.0 +* All related operations, including memory management, are handled by the library. +* */ + + +/************************************** +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/************************************** +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(s) calloc(1,s) +#define FREEMEM free +#include /* memset, memcpy, memmove */ +#define MEM_INIT memset + + +/************************************** +* Includes +**************************************/ +#include "lz4frame_static.h" +#include "lz4.h" +#include "lz4hc.h" +#include "xxhash.h" + + +/************************************** +* Basic Types +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + + +/************************************** +* Constants +**************************************/ +#define KB *(1<<10) +#define MB *(1<<20) +#define GB *(1<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LZ4F_MAGIC_SKIPPABLE_START 0x184D2A50U +#define LZ4F_MAGICNUMBER 0x184D2204U +#define LZ4F_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LZ4F_BLOCKSIZEID_DEFAULT LZ4F_max64KB + +static const size_t minFHSize = 7; +static const size_t maxFHSize = 15; +static const size_t BHSize = 4; +static const int minHClevel = 3; + + +/************************************** +* Structures and local types +**************************************/ +typedef struct LZ4F_cctx_s +{ + LZ4F_preferences_t prefs; + U32 version; + U32 cStage; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpBuff; + BYTE* tmpIn; + size_t tmpInSize; + U64 totalInSize; + XXH32_state_t xxh; + void* lz4CtxPtr; + U32 lz4CtxLevel; /* 0: unallocated; 1: LZ4_stream_t; 3: LZ4_streamHC_t */ +} LZ4F_cctx_t; + +typedef struct LZ4F_dctx_s +{ + LZ4F_frameInfo_t frameInfo; + U32 version; + U32 dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + const BYTE* srcExpect; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + BYTE header[16]; +} LZ4F_dctx_t; + + +/************************************** +* Error management +**************************************/ +#define LZ4F_GENERATE_STRING(STRING) #STRING, +static const char* LZ4F_errorStrings[] = { LZ4F_LIST_ERRORS(LZ4F_GENERATE_STRING) }; + + +unsigned LZ4F_isError(LZ4F_errorCode_t code) +{ + return (code > (LZ4F_errorCode_t)(-LZ4F_ERROR_maxCode)); +} + +const char* LZ4F_getErrorName(LZ4F_errorCode_t code) +{ + static const char* codeError = "Unspecified error code"; + if (LZ4F_isError(code)) return LZ4F_errorStrings[-(int)(code)]; + return codeError; +} + + +/************************************** +* Private functions +**************************************/ +static size_t LZ4F_getBlockSize(unsigned blockSizeID) +{ + static const size_t blockSizes[4] = { 64 KB, 256 KB, 1 MB, 4 MB }; + + if (blockSizeID == 0) blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + blockSizeID -= 4; + if (blockSizeID > 3) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; + return blockSizes[blockSizeID]; +} + + +/* unoptimized version; solves endianess & alignment issues */ +static U32 LZ4F_readLE32 (const BYTE* srcPtr) +{ + U32 value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += ((U32)srcPtr[3])<<24; + return value32; +} + +static void LZ4F_writeLE32 (BYTE* dstPtr, U32 value32) +{ + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + +static U64 LZ4F_readLE64 (const BYTE* srcPtr) +{ + U64 value64 = srcPtr[0]; + value64 += ((U64)srcPtr[1]<<8); + value64 += ((U64)srcPtr[2]<<16); + value64 += ((U64)srcPtr[3]<<24); + value64 += ((U64)srcPtr[4]<<32); + value64 += ((U64)srcPtr[5]<<40); + value64 += ((U64)srcPtr[6]<<48); + value64 += ((U64)srcPtr[7]<<56); + return value64; +} + +static void LZ4F_writeLE64 (BYTE* dstPtr, U64 value64) +{ + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); +} + + +static BYTE LZ4F_headerChecksum (const void* header, size_t length) +{ + U32 xxh = XXH32(header, length, 0); + return (BYTE)(xxh >> 8); +} + + +/************************************** +* Simple compression functions +**************************************/ +static LZ4F_blockSizeID_t LZ4F_optimalBSID(const LZ4F_blockSizeID_t requestedBSID, const size_t srcSize) +{ + LZ4F_blockSizeID_t proposedBSID = LZ4F_max64KB; + size_t maxBlockSize = 64 KB; + while (requestedBSID > proposedBSID) + { + if (srcSize <= maxBlockSize) + return proposedBSID; + proposedBSID = (LZ4F_blockSizeID_t)((int)proposedBSID + 1); + maxBlockSize <<= 2; + } + return requestedBSID; +} + + +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefs; + size_t headerSize; + size_t streamSize; + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + else memset(&prefs, 0, sizeof(prefs)); + + prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + + headerSize = maxFHSize; /* header size, including magic number and frame content size*/ + streamSize = LZ4F_compressBound(srcSize, &prefs); + + return headerSize + streamSize; +} + + +/* LZ4F_compressFrame() +* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.0, in a single step. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() +* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) +* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. +* The result of the function is the number of bytes written into dstBuffer. +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +*/ +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_cctx_t cctxI; + LZ4_stream_t lz4ctx; + LZ4F_preferences_t prefs; + LZ4F_compressOptions_t options; + LZ4F_errorCode_t errorCode; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* const dstEnd = dstStart + dstMaxSize; + + memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ + memset(&options, 0, sizeof(options)); + + cctxI.version = LZ4F_VERSION; + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ + + if (preferencesPtr!=NULL) + prefs = *preferencesPtr; + else + memset(&prefs, 0, sizeof(prefs)); + if (prefs.frameInfo.contentSize != 0) + prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ + + if (prefs.compressionLevel < (int)minHClevel) + { + cctxI.lz4CtxPtr = &lz4ctx; + cctxI.lz4CtxLevel = 1; + } + + prefs.frameInfo.blockSizeID = LZ4F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + if (srcSize <= LZ4F_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = LZ4F_blockIndependent; /* no need for linked blocks */ + + options.stableSrc = 1; + + if (dstMaxSize < LZ4F_compressFrameBound(srcSize, &prefs)) + return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; + + errorCode = LZ4F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; /* header size */ + + errorCode = LZ4F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + errorCode = LZ4F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + if (prefs.compressionLevel >= (int)minHClevel) /* no allocation necessary with lz4 fast */ + FREEMEM(cctxI.lz4CtxPtr); + + return (dstPtr - dstStart); +} + + +/*********************************** +* Advanced compression functions +***********************************/ + +/* LZ4F_createCompressionContext() : +* The first thing to do is to create a compressionContext object, which will be used in all compression operations. +* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. +* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. +* The function will provide a pointer to an allocated LZ4F_compressionContext_t object. +* If the result LZ4F_errorCode_t is not OK_NoError, there was an error during context creation. +* Object can release its memory using LZ4F_freeCompressionContext(); +*/ +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* LZ4F_compressionContextPtr, unsigned version) +{ + LZ4F_cctx_t* cctxPtr; + + cctxPtr = (LZ4F_cctx_t*)ALLOCATOR(sizeof(LZ4F_cctx_t)); + if (cctxPtr==NULL) return (LZ4F_errorCode_t)(-LZ4F_ERROR_allocation_failed); + + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Next stage : write header */ + + *LZ4F_compressionContextPtr = (LZ4F_compressionContext_t)cctxPtr; + + return LZ4F_OK_NoError; +} + + +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t LZ4F_compressionContext) +{ + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)LZ4F_compressionContext; + + if (cctxPtr != NULL) /* null pointers can be safely provided to this function, like free() */ + { + FREEMEM(cctxPtr->lz4CtxPtr); + FREEMEM(cctxPtr->tmpBuff); + FREEMEM(LZ4F_compressionContext); + } + + return LZ4F_OK_NoError; +} + + +/* LZ4F_compressBegin() : +* will write the frame header into dstBuffer. +* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ4F_MAXHEADERFRAME_SIZE bytes. +* The result of the function is the number of bytes written into dstBuffer for the header +* or an error code (can be tested using LZ4F_isError()) +*/ +size_t LZ4F_compressBegin(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefNull; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* headerStart; + size_t requiredBuffSize; + + if (dstMaxSize < maxFHSize) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return (size_t)-LZ4F_ERROR_GENERIC; + memset(&prefNull, 0, sizeof(prefNull)); + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctxPtr->prefs = *preferencesPtr; + + /* ctx Management */ + { + U32 tableID = (cctxPtr->prefs.compressionLevel < minHClevel) ? 1 : 2; /* 0:nothing ; 1:LZ4 table ; 2:HC tables */ + if (cctxPtr->lz4CtxLevel < tableID) + { + FREEMEM(cctxPtr->lz4CtxPtr); + if (cctxPtr->prefs.compressionLevel < minHClevel) + cctxPtr->lz4CtxPtr = (void*)LZ4_createStream(); + else + cctxPtr->lz4CtxPtr = (void*)LZ4_createStreamHC(); + cctxPtr->lz4CtxLevel = tableID; + } + } + + /* Buffer Management */ + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ4F_BLOCKSIZEID_DEFAULT; + cctxPtr->maxBlockSize = LZ4F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); + + requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 128 KB); + if (preferencesPtr->autoFlush) + requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ4F_blockLinked) * 64 KB; /* just needs dict */ + + if (cctxPtr->maxBufferSize < requiredBuffSize) + { + cctxPtr->maxBufferSize = requiredBuffSize; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) return (size_t)-LZ4F_ERROR_allocation_failed; + } + cctxPtr->tmpIn = cctxPtr->tmpBuff; + cctxPtr->tmpInSize = 0; + XXH32_reset(&(cctxPtr->xxh), 0); + if (cctxPtr->prefs.compressionLevel < minHClevel) + LZ4_resetStream((LZ4_stream_t*)(cctxPtr->lz4CtxPtr)); + else + LZ4_resetStreamHC((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), cctxPtr->prefs.compressionLevel); + + /* Magic Number */ + LZ4F_writeLE32(dstPtr, LZ4F_MAGICNUMBER); + dstPtr += 4; + headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ + /* BD Byte */ + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.contentSize) + { + LZ4F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } + /* CRC Byte */ + *dstPtr = LZ4F_headerChecksum(headerStart, dstPtr - headerStart); + dstPtr++; + + cctxPtr->cStage = 1; /* header written, now request input data block */ + + return (dstPtr - dstStart); +} + + +/* LZ4F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. +* The LZ4F_frameInfo_t structure is optional : +* you can provide NULL as argument, preferences will then be set to cover worst case situations. +* */ +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr) +{ + LZ4F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + prefsNull.frameInfo.contentChecksumFlag = LZ4F_contentChecksumEnabled; /* worst case */ + { + const LZ4F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LZ4F_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LZ4F_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + } +} + + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level); + +static size_t LZ4F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz4ctx, int level) +{ + /* compress one block */ + BYTE* cSizePtr = (BYTE*)dst; + U32 cSize; + cSize = (U32)compress(lz4ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1), level); + LZ4F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = (U32)srcSize; + LZ4F_writeLE32(cSizePtr, cSize + LZ4F_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+4, src, srcSize); + } + return cSize + 4; +} + + +static int LZ4F_localLZ4_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); +} + +static int LZ4F_localLZ4_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_limitedOutput_continue((LZ4_stream_t*)ctx, src, dst, srcSize, dstSize); +} + +static int LZ4F_localLZ4_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize, int level) +{ + (void) level; + return LZ4_compress_HC_continue((LZ4_streamHC_t*)ctx, src, dst, srcSize, dstSize); +} + +static compressFunc_t LZ4F_selectCompression(LZ4F_blockMode_t blockMode, int level) +{ + if (level < minHClevel) + { + if (blockMode == LZ4F_blockIndependent) return LZ4F_localLZ4_compress_limitedOutput_withState; + return LZ4F_localLZ4_compress_limitedOutput_continue; + } + if (blockMode == LZ4F_blockIndependent) return LZ4_compress_HC_extStateHC; + return LZ4F_localLZ4_compressHC_limitedOutput_continue; +} + +static int LZ4F_localSaveDict(LZ4F_cctx_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < minHClevel) + return LZ4_saveDict ((LZ4_stream_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ4_saveDictHC ((LZ4_streamHC_t*)(cctxPtr->lz4CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); +} + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ4F_lastBlockStatus; + +/* LZ4F_compressUpdate() +* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode) +* You can get the minimum value of dstMaxSize by using LZ4F_compressBound() +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +*/ +size_t LZ4F_compressUpdate(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_compressOptions_t cOptionsNull; + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; + size_t blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + LZ4F_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t compress; + + + if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; + if (dstMaxSize < LZ4F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + + /* select compression function */ + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + + /* complete tmp buffer */ + if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ + { + size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; + if (sizeToCopy > srcSize) + { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } + else + { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } + } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) + { + /* compress full block */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + srcPtr += blockSize; + } + + if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) + { + /* compress remaining input < blockSize */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ4F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + srcPtr = srcEnd; + } + + /* preserve dictionary if necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) + { + if (compressOptionsPtr->stableSrc) + { + cctxPtr->tmpIn = cctxPtr->tmpBuff; + } + else + { + int realDictSize = LZ4F_localSaveDict(cctxPtr); + if (realDictSize==0) return (size_t)-LZ4F_ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ4F_blockLinked && lastBlockCompressed==fromTmpBuffer */ + && !(cctxPtr->prefs.autoFlush)) + { + int realDictSize = LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) + { + /* fill tmp buffer */ + size_t sizeToCopy = srcEnd - srcPtr; + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) + XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + + cctxPtr->totalInSize += srcSize; + return dstPtr - dstStart; +} + + +/* LZ4F_flush() +* Should you need to create compressed data immediately, without waiting for a block to be filled, +* you can call LZ4_flush(), which will immediately compress any remaining data stored within compressionContext. +* The result of the function is the number of bytes written into dstBuffer +* (it can be zero, this means there was no data left within compressionContext) +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +*/ +size_t LZ4F_flush(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + compressFunc_t compress; + + + if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ + if (cctxPtr->cStage != 1) return (size_t)-LZ4F_ERROR_GENERIC; + if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LZ4F_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ + (void)compressOptionsPtr; /* not yet useful */ + + /* select compression function */ + compress = LZ4F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + + /* compress tmp buffer */ + dstPtr += LZ4F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz4CtxPtr, cctxPtr->prefs.compressionLevel); + if (cctxPtr->prefs.frameInfo.blockMode==LZ4F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily LZ4F_blockLinked */ + { + int realDictSize = LZ4F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + return dstPtr - dstStart; +} + + +/* LZ4F_compressEnd() +* When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). +* It will flush whatever data remained within compressionContext (like LZ4_flush()) +* but also properly finalize the frame, with an endMark and a checksum. +* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) +* The function outputs an error code if it fails (can be tested using LZ4F_isError()) +* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. +* compressionContext can then be used again, starting with LZ4F_compressBegin(). The preferences will remain the same. +*/ +size_t LZ4F_compressEnd(LZ4F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* compressOptionsPtr) +{ + LZ4F_cctx_t* cctxPtr = (LZ4F_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + size_t errorCode; + + errorCode = LZ4F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LZ4F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + LZ4F_writeLE32(dstPtr, 0); + dstPtr+=4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ4F_contentChecksumEnabled) + { + U32 xxh = XXH32_digest(&(cctxPtr->xxh)); + LZ4F_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + + if (cctxPtr->prefs.frameInfo.contentSize) + { + if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) + return (size_t)-LZ4F_ERROR_frameSize_wrong; + } + + return dstPtr - dstStart; +} + + +/********************************** +* Decompression functions +**********************************/ + +/* Resource management */ + +/* LZ4F_createDecompressionContext() : +* The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. +* This is achieved using LZ4F_createDecompressionContext(). +* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext object. +* If the result LZ4F_errorCode_t is not zero, there was an error during context creation. +* Object can release its memory using LZ4F_freeDecompressionContext(); +*/ +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* LZ4F_decompressionContextPtr, unsigned versionNumber) +{ + LZ4F_dctx_t* dctxPtr; + + dctxPtr = (LZ4F_dctx_t*)ALLOCATOR(sizeof(LZ4F_dctx_t)); + if (dctxPtr==NULL) return (LZ4F_errorCode_t)-LZ4F_ERROR_GENERIC; + + dctxPtr->version = versionNumber; + *LZ4F_decompressionContextPtr = (LZ4F_decompressionContext_t)dctxPtr; + return LZ4F_OK_NoError; +} + +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t LZ4F_decompressionContext) +{ + LZ4F_errorCode_t result = LZ4F_OK_NoError; + LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)LZ4F_decompressionContext; + if (dctxPtr != NULL) /* can accept NULL input, like free() */ + { + result = (LZ4F_errorCode_t)dctxPtr->dStage; + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + FREEMEM(dctxPtr); + } + return result; +} + + +/* ******************************************************************** */ +/* ********************* Decompression ******************************** */ +/* ******************************************************************** */ + +typedef enum { dstage_getHeader=0, dstage_storeHeader, + dstage_getCBlockSize, dstage_storeCBlockSize, + dstage_copyDirect, + dstage_getCBlock, dstage_storeCBlock, + dstage_decodeCBlock, dstage_decodeCBlock_intoDst, + dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + + +/* LZ4F_decodeHeader + return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) + or an error code (testable with LZ4F_isError()) + output : set internal values of dctx, such as + dctxPtr->frameInfo and dctxPtr->dStage. + input : srcVoidPtr points at the **beginning of the frame** +*/ +static size_t LZ4F_decodeHeader(LZ4F_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) +{ + BYTE FLG, BD, HC; + unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; + size_t bufferNeeded; + size_t frameHeaderSize; + const BYTE* srcPtr = (const BYTE*)srcVoidPtr; + + /* need to decode header to get frameInfo */ + if (srcSize < minFHSize) return (size_t)-LZ4F_ERROR_frameHeader_incomplete; /* minimal frame header size */ + memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); + + /* special case : skippable frames */ + if ((LZ4F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ4F_MAGIC_SKIPPABLE_START) + { + dctxPtr->frameInfo.frameType = LZ4F_skippableFrame; + if (srcVoidPtr == (void*)(dctxPtr->header)) + { + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + return srcSize; + } + else + { + dctxPtr->dStage = dstage_getSFrameSize; + return 4; + } + } + + /* control magic number */ + if (LZ4F_readLE32(srcPtr) != LZ4F_MAGICNUMBER) return (size_t)-LZ4F_ERROR_frameType_unknown; + dctxPtr->frameInfo.frameType = LZ4F_frame; + + /* Flags */ + FLG = srcPtr[4]; + version = (FLG>>6) & _2BITS; + blockMode = (FLG>>5) & _1BIT; + blockChecksumFlag = (FLG>>4) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + + /* Frame Header Size */ + frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; + + if (srcSize < frameHeaderSize) + { + /* not enough input to fully decode frame header */ + if (srcPtr != dctxPtr->header) + memcpy(dctxPtr->header, srcPtr, srcSize); + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = frameHeaderSize; + dctxPtr->dStage = dstage_storeHeader; + return srcSize; + } + + BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; + + /* validate */ + if (version != 1) return (size_t)-LZ4F_ERROR_headerVersion_wrong; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-LZ4F_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */ + if (((BD>>7)&_1BIT) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bit */ + if (blockSizeID < 4) return (size_t)-LZ4F_ERROR_maxBlockSize_invalid; /* 4-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-LZ4F_ERROR_reservedFlag_set; /* Reserved bits */ + + /* check */ + HC = LZ4F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LZ4F_ERROR_headerChecksum_invalid; /* Bad header checksum error */ + + /* save */ + dctxPtr->frameInfo.blockMode = (LZ4F_blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (LZ4F_contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (LZ4F_blockSizeID_t)blockSizeID; + dctxPtr->maxBlockSize = LZ4F_getBlockSize(blockSizeID); + if (contentSizeFlag) + dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ4F_readLE64(srcPtr+6); + + /* init */ + if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); + + /* alloc */ + bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) * 128 KB); + if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */ + { + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + dctxPtr->maxBufferSize = bufferNeeded; + dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); + if (dctxPtr->tmpIn == NULL) return (size_t)-LZ4F_ERROR_GENERIC; + dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); + if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LZ4F_ERROR_GENERIC; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = 0; + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = 0; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; + dctxPtr->tmpOutStart = 0; + dctxPtr->tmpOutSize = 0; + + dctxPtr->dStage = dstage_getCBlockSize; + + return frameHeaderSize; +} + + +/* LZ4F_getFrameInfo() +* This function decodes frame header information, such as blockSize. +* It is optional : you could start by calling directly LZ4F_decompress() instead. +* The objective is to extract header information without starting decompression, typically for allocation purposes. +* LZ4F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ4F_decompressionContext_t. +* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) +* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress, +* or an error code which can be tested using LZ4F_isError(). +*/ +LZ4F_errorCode_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dCtx, LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) +{ + LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)dCtx; + + if (dctxPtr->dStage > dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ + { + size_t o=0, i=0; + /* frameInfo already decoded */ + *srcSizePtr = 0; + *frameInfoPtr = dctxPtr->frameInfo; + return LZ4F_decompress(dCtx, NULL, &o, NULL, &i, NULL); + } + else + { + size_t o=0; + size_t nextSrcSize = LZ4F_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); + if (dctxPtr->dStage <= dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ + return (size_t)-LZ4F_ERROR_frameHeader_incomplete; + *frameInfoPtr = dctxPtr->frameInfo; + return nextSrcSize; + } +} + + +/* trivial redirector, for common prototype */ +static int LZ4F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +{ + (void)dictStart; (void)dictSize; + return LZ4_decompress_safe (source, dest, compressedSize, maxDecompressedSize); +} + + +static void LZ4F_updateDict(LZ4F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +{ + if (dctxPtr->dictSize==0) + dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ + + if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */ + { + dctxPtr->dictSize += dstSize; + return; + } + + if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */ + { + dctxPtr->dict = (const BYTE*)dstPtr0; + dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; + return; + } + + if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) + { + /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ + dctxPtr->dictSize += dstSize; + return; + } + + if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ + { + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = 64 KB - dctxPtr->tmpOutSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; + return; + } + + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ + { + if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */ + { + size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + dctxPtr->dictSize = preserveSize; + } + memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); + dctxPtr->dictSize += dstSize; + return; + } + + /* join dict & dest into tmp */ + { + size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dstSize; + } +} + + + +/* LZ4F_decompress() +* Call this function repetitively to regenerate data compressed within srcBuffer. +* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. +* +* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). +* +* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. +* You will have to call it again, continuing from where it stopped. +* +* The function result is an hint of the better srcSize to use for next call to LZ4F_decompress. +* Basically, it's the size of the current (or remaining) compressed block + header of next block. +* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. +* Note that this is just a hint, you can always provide any srcSize you want. +* When a frame is fully decoded, the function result will be 0. +* If decompression failed, function result is an error code which can be tested using LZ4F_isError(). +*/ +size_t LZ4F_decompress(LZ4F_decompressionContext_t decompressionContext, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* decompressOptionsPtr) +{ + LZ4F_dctx_t* dctxPtr = (LZ4F_dctx_t*)decompressionContext; + LZ4F_decompressOptions_t optionsNull; + const BYTE* const srcStart = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcStart + *srcSizePtr; + const BYTE* srcPtr = srcStart; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* const dstEnd = dstStart + *dstSizePtr; + BYTE* dstPtr = dstStart; + const BYTE* selectedIn = NULL; + unsigned doAnotherStage = 1; + size_t nextSrcSizeHint = 1; + + + memset(&optionsNull, 0, sizeof(optionsNull)); + if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; + *srcSizePtr = 0; + *dstSizePtr = 0; + + /* expect to continue decoding src buffer where it left previously */ + if (dctxPtr->srcExpect != NULL) + { + if (srcStart != dctxPtr->srcExpect) return (size_t)-LZ4F_ERROR_srcPtr_wrong; + } + + /* programmed as a state machine */ + + while (doAnotherStage) + { + + switch(dctxPtr->dStage) + { + + case dstage_getHeader: + { + if ((size_t)(srcEnd-srcPtr) >= maxFHSize) /* enough to decode - shortcut */ + { + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); + if (LZ4F_isError(errorCode)) return errorCode; + srcPtr += errorCode; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ + dctxPtr->dStage = dstage_storeHeader; + } + + case dstage_storeHeader: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) + { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ + break; + } + { + LZ4F_errorCode_t errorCode = LZ4F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); + if (LZ4F_isError(errorCode)) return errorCode; + } + break; + } + + case dstage_getCBlockSize: + { + if ((size_t)(srcEnd - srcPtr) >= BHSize) + { + selectedIn = srcPtr; + srcPtr += BHSize; + } + else + { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; + } + } + + if (dctxPtr->dStage == dstage_storeCBlockSize) + case dstage_storeCBlockSize: + { + size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < BHSize) /* not enough input to get full cBlockSize; wait for more */ + { + nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->tmpIn; + } + + /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ + { + size_t nextCBlockSize = LZ4F_readLE32(selectedIn) & 0x7FFFFFFFU; + if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ + { + dctxPtr->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LZ4F_ERROR_GENERIC; /* invalid cBlockSize */ + dctxPtr->tmpInTarget = nextCBlockSize; + if (LZ4F_readLE32(selectedIn) & LZ4F_BLOCKUNCOMPRESSED_FLAG) + { + dctxPtr->dStage = dstage_copyDirect; + break; + } + dctxPtr->dStage = dstage_getCBlock; + if (dstPtr==dstEnd) + { + nextSrcSizeHint = nextCBlockSize + BHSize; + doAnotherStage = 0; + } + break; + } + + case dstage_copyDirect: /* uncompressed block */ + { + size_t sizeToCopy = dctxPtr->tmpInTarget; + if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ + if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + memcpy(dstPtr, srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ + nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; + doAnotherStage = 0; + break; + } + + case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ + { + if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlock; + break; + } + selectedIn = srcPtr; + srcPtr += dctxPtr->tmpInTarget; + dctxPtr->dStage = dstage_decodeCBlock; + break; + } + + case dstage_storeCBlock: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ + { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlock; + break; + } + + case dstage_decodeCBlock: + { + if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + else + dctxPtr->dStage = dstage_decodeCBlock_intoDst; + break; + } + + case dstage_decodeCBlock_intoDst: + { + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; + + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-LZ4F_ERROR_GENERIC; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + + dstPtr += decodedSize; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + + case dstage_decodeCBlock_intoTmp: + { + /* not enough place into dst : decode into tmpOut */ + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) + decoder = LZ4_decompress_safe_usingDict; + else + decoder = LZ4F_decompress_safe; + + /* ensure enough place for tmpOut */ + if (dctxPtr->frameInfo.blockMode == LZ4F_blockLinked) + { + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) + { + if (dctxPtr->dictSize > 128 KB) + { + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); + dctxPtr->dictSize = 64 KB; + } + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; + } + else /* dict not within tmp */ + { + size_t reservedDictSpace = dctxPtr->dictSize; + if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; + } + } + + /* Decode */ + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-LZ4F_ERROR_decompressionFailed; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + dctxPtr->tmpOutSize = decodedSize; + dctxPtr->tmpOutStart = 0; + dctxPtr->dStage = dstage_flushOut; + break; + } + + case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ + { + size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; + if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; + memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) + LZ4F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + + dctxPtr->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; + + /* end of flush ? */ + if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + nextSrcSizeHint = BHSize; + doAnotherStage = 0; /* still some data to flush */ + break; + } + + case dstage_getSuffix: + { + size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; + if (dctxPtr->frameRemainingSize) return (size_t)-LZ4F_ERROR_frameSize_wrong; /* incorrect frame size decoded */ + if (suffixSize == 0) /* frame completed */ + { + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) /* not enough size for entire CRC */ + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; + } + else + { + selectedIn = srcPtr; + srcPtr += 4; + } + } + + if (dctxPtr->dStage == dstage_storeSuffix) + case dstage_storeSuffix: + { + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ + { + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + } + + /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ + { + U32 readCRC = LZ4F_readLE32(selectedIn); + U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); + if (readCRC != resultCRC) return (size_t)-LZ4F_ERROR_contentChecksum_invalid; + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + + case dstage_getSFrameSize: + { + if ((srcEnd - srcPtr) >= 4) + { + selectedIn = srcPtr; + srcPtr += 4; + } + else + { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 4; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + } + } + + if (dctxPtr->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */ + { + nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->header + 4; + } + + /* case dstage_decodeSFrameSize: */ /* no direct access */ + { + size_t SFrameSize = LZ4F_readLE32(selectedIn); + dctxPtr->frameInfo.contentSize = SFrameSize; + dctxPtr->tmpInTarget = SFrameSize; + dctxPtr->dStage = dstage_skipSkippable; + break; + } + + case dstage_skipSkippable: + { + size_t skipSize = dctxPtr->tmpInTarget; + if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + srcPtr += skipSize; + dctxPtr->tmpInTarget -= skipSize; + doAnotherStage = 0; + nextSrcSizeHint = dctxPtr->tmpInTarget; + if (nextSrcSizeHint) break; + dctxPtr->dStage = dstage_getHeader; + break; + } + } + } + + /* preserve dictionary within tmp if necessary */ + if ( (dctxPtr->frameInfo.blockMode==LZ4F_blockLinked) + &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) + &&(!decompressOptionsPtr->stableDst) + &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) + ) + { + if (dctxPtr->dStage == dstage_flushOut) + { + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = 64 KB - dctxPtr->tmpOutSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; + } + else + { + size_t newDictSize = dctxPtr->dictSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + if ((newDictSize) > 64 KB) newDictSize = 64 KB; + + memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = newDictSize; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + } + } + + /* require function to be called again from position where it stopped */ + if (srcPtrsrcExpect = srcPtr; + else + dctxPtr->srcExpect = NULL; + + *srcSizePtr = (srcPtr - srcStart); + *dstSizePtr = (dstPtr - dstStart); + return nextSrcSizeHint; +} diff --git a/C/lz4/lz4frame.h b/C/lz4/lz4frame.h new file mode 100644 index 00000000..05fbc5fa --- /dev/null +++ b/C/lz4/lz4frame.h @@ -0,0 +1,303 @@ +/* + LZ4 auto-framing library + Header File + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +/* LZ4F is a stand-alone API to create LZ4-compressed frames + * fully conformant to specification v1.5.1. + * All related operations, including memory management, are handled by the library. + * You don't need lz4.h when using lz4frame.h. + * */ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/************************************** +* Includes +**************************************/ +#include /* size_t */ + + +/************************************** + * Error management + * ************************************/ +typedef size_t LZ4F_errorCode_t; + +unsigned LZ4F_isError(LZ4F_errorCode_t code); +const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */ + + +/************************************** + * Frame compression types + * ************************************/ +//#define LZ4F_DISABLE_OBSOLETE_ENUMS +#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +# define LZ4F_OBSOLETE_ENUM(x) ,x +#else +# define LZ4F_OBSOLETE_ENUM(x) +#endif + +typedef enum { + LZ4F_default=0, + LZ4F_max64KB=4, + LZ4F_max256KB=5, + LZ4F_max1MB=6, + LZ4F_max4MB=7 + LZ4F_OBSOLETE_ENUM(max64KB = LZ4F_max64KB) + LZ4F_OBSOLETE_ENUM(max256KB = LZ4F_max256KB) + LZ4F_OBSOLETE_ENUM(max1MB = LZ4F_max1MB) + LZ4F_OBSOLETE_ENUM(max4MB = LZ4F_max4MB) +} LZ4F_blockSizeID_t; + +typedef enum { + LZ4F_blockLinked=0, + LZ4F_blockIndependent + LZ4F_OBSOLETE_ENUM(blockLinked = LZ4F_blockLinked) + LZ4F_OBSOLETE_ENUM(blockIndependent = LZ4F_blockIndependent) +} LZ4F_blockMode_t; + +typedef enum { + LZ4F_noContentChecksum=0, + LZ4F_contentChecksumEnabled + LZ4F_OBSOLETE_ENUM(noContentChecksum = LZ4F_noContentChecksum) + LZ4F_OBSOLETE_ENUM(contentChecksumEnabled = LZ4F_contentChecksumEnabled) +} LZ4F_contentChecksum_t; + +typedef enum { + LZ4F_frame=0, + LZ4F_skippableFrame + LZ4F_OBSOLETE_ENUM(skippableFrame = LZ4F_skippableFrame) +} LZ4F_frameType_t; + +#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS +typedef LZ4F_blockSizeID_t blockSizeID_t; +typedef LZ4F_blockMode_t blockMode_t; +typedef LZ4F_frameType_t frameType_t; +typedef LZ4F_contentChecksum_t contentChecksum_t; +#endif + +typedef struct { + LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */ + unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ + unsigned reserved[2]; /* must be zero for forward compatibility */ +} LZ4F_frameInfo_t; + +typedef struct { + LZ4F_frameInfo_t frameInfo; + int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ + unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ + unsigned reserved[4]; /* must be zero for forward compatibility */ +} LZ4F_preferences_t; + + +/*********************************** + * Simple compression function + * *********************************/ +size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr); + +size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr); +/* LZ4F_compressFrame() + * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1 + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound() + * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode) + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * The result of the function is the number of bytes written into dstBuffer. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ + + + +/********************************** +* Advanced compression functions +**********************************/ +typedef struct LZ4F_cctx_s* LZ4F_compressionContext_t; /* must be aligned on 8-bytes */ + +typedef struct { + unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned reserved[3]; +} LZ4F_compressOptions_t; + +/* Resource Management */ + +#define LZ4F_VERSION 100 +LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* cctxPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t cctx); +/* LZ4F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure. + * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object. + * If the result LZ4F_errorCode_t is not zero, there was an error during context creation. + * Object can release its memory using LZ4F_freeCompressionContext(); + */ + + +/* Compression */ + +size_t LZ4F_compressBegin(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* prefsPtr); +/* LZ4F_compressBegin() : + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. + * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The result of the function is the number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ4F_isError()) + */ + +size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr); +/* LZ4F_compressBound() : + * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. + * Different preferences can produce different results. + * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. + * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + */ + +size_t LZ4F_compressUpdate(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr); +/* LZ4F_compressUpdate() + * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ4F_compressBound(). + * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + * LZ4F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ + +size_t LZ4F_flush(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +/* LZ4F_flush() + * Should you need to generate compressed data immediately, without waiting for the current block to be filled, + * you can call LZ4_flush(), which will immediately compress any remaining data buffered within cctx. + * Note that dstMaxSize must be large enough to ensure the operation will be successful. + * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within cctx) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + */ + +size_t LZ4F_compressEnd(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr); +/* LZ4F_compressEnd() + * When you want to properly finish the compressed frame, just call LZ4F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ4_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) + * The function outputs an error code if it fails (can be tested using LZ4F_isError()) + * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument. + * A successful call to LZ4F_compressEnd() makes cctx available again for next compression task. + */ + + +/*********************************** +* Decompression functions +***********************************/ + +typedef struct LZ4F_dctx_s* LZ4F_decompressionContext_t; /* must be aligned on 8-bytes */ + +typedef struct { + unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ + unsigned reserved[3]; +} LZ4F_decompressOptions_t; + + +/* Resource management */ + +LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* dctxPtr, unsigned version); +LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t dctx); +/* LZ4F_createDecompressionContext() : + * The first thing to do is to create an LZ4F_decompressionContext_t object, which will be used in all decompression operations. + * This is achieved using LZ4F_createDecompressionContext(). + * The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions. + * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object. + * The result is an errorCode, which can be tested using LZ4F_isError(). + * dctx memory can be released using LZ4F_freeDecompressionContext(); + * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + * That is, it should be == 0 if decompression has been completed fully and correctly. + */ + + +/* Decompression */ + +size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dctx, + LZ4F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); +/* LZ4F_getFrameInfo() + * This function decodes frame header information (such as max blockSize, frame checksum, etc.). + * Its usage is optional : you can start by calling directly LZ4F_decompress() instead. + * The objective is to extract frame header information, typically for allocation purposes. + * LZ4F_getFrameInfo() can also be used anytime *after* starting decompression, on any valid LZ4F_decompressionContext_t. + * The result is *copied* into an existing LZ4F_frameInfo_t structure which must be already allocated. + * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call, + * or an error code which can be tested using LZ4F_isError() + * (typically, when there is not enough src bytes to fully decode the frame header) + * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) + */ + +size_t LZ4F_decompress(LZ4F_decompressionContext_t dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ4F_decompressOptions_t* dOptPtr); +/* LZ4F_decompress() + * Call this function repetitively to regenerate data compressed within srcBuffer. + * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not completed. + * It typically happens when dstBuffer is not large enough to contain all decoded data. + * LZ4F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) + * The function will check this condition, and refuse to continue if it is not respected. + * + * dstBuffer is supposed to be flushed between each call to the function, since its content will be overwritten. + * dst arguments can be changed at will with each consecutive call to the function. + * + * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. + * This is just a hint, you can always provide any srcSize you want. + * When a frame is fully decoded, the function result will be 0 (no more data expected). + * If decompression failed, function result is an error code, which can be tested using LZ4F_isError(). + * + * After a frame is fully decoded, dctx can be used again to decompress another frame. + */ + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz4/lz4frame_static.h b/C/lz4/lz4frame_static.h new file mode 100644 index 00000000..0d909753 --- /dev/null +++ b/C/lz4/lz4frame_static.h @@ -0,0 +1,81 @@ +/* + LZ4 auto-framing library + Header File for static linking only + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* lz4frame_static.h should be used solely in the context of static linking. + * It contains definitions which may still change overtime. + * Never use it in the context of DLL linking. + * */ + + +/************************************** +* Includes +**************************************/ +#include "lz4frame.h" + + +/************************************** + * Error management + * ************************************/ +#define LZ4F_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_maxCode) + +//#define LZ4F_DISABLE_OLD_ENUMS +#ifndef LZ4F_DISABLE_OLD_ENUMS +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM, +#else +#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, +#endif +typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz4/lz4hc.c b/C/lz4/lz4hc.c new file mode 100644 index 00000000..bbe7a9d2 --- /dev/null +++ b/C/lz4/lz4hc.c @@ -0,0 +1,731 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ + + + +/************************************** +* Tuning Parameter +**************************************/ +static const int LZ4HC_compressionLevel_default = 9; + + +/************************************** +* Includes +**************************************/ +#include "lz4hc.h" + + +/************************************** +* Local Compiler Options +**************************************/ +#if defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +#if defined (__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#endif + + +/************************************** +* Common LZ4 definition +**************************************/ +#define LZ4_COMMONDEFS_ONLY +#include "lz4.c" + + +/************************************** +* Local Constants +**************************************/ +#define DICTIONARY_LOGSIZE 16 +#define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) +//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */ +#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */ + +static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); } + + + +/************************************** +* HC Compression +**************************************/ +static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start) +{ + MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); + hc4->nextToUpdate = 64 KB; + hc4->base = start - 64 KB; + hc4->end = start; + hc4->dictBase = start - 64 KB; + hc4->dictLimit = 64 KB; + hc4->lowLimit = 64 KB; +} + + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) +{ + U16* chainTable = hc4->chainTable; + U32* HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const U32 target = (U32)(ip - base); + U32 idx = hc4->nextToUpdate; + + while(idx < target) + { + U32 h = LZ4HC_hashPtr(base+idx); + size_t delta = idx - HashTable[h]; + if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; + DELTANEXTU16(idx) = (U16)delta; + HashTable[h] = idx; + idx++; + } + + hc4->nextToUpdate = target; +} + + +FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos, + const int maxNbAttempts) +{ + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const BYTE* const dictBase = hc4->dictBase; + const U32 dictLimit = hc4->dictLimit; + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + U32 matchIndex; + const BYTE* match; + int nbAttempts=maxNbAttempts; + size_t ml=0; + + /* HC4 match finder */ + LZ4HC_Insert(hc4, ip); + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + + while ((matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) + && (LZ4_read32(match) == LZ4_read32(ip))) + { + size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (LZ4_read32(match) == LZ4_read32(ip)) + { + size_t mlt; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit); + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + matchIndex -= DELTANEXTU16(matchIndex); + } + + return (int)ml; +} + + +FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch ( + LZ4HC_Data_Structure* hc4, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + int longest, + const BYTE** matchpos, + const BYTE** startpos, + const int maxNbAttempts) +{ + U16* const chainTable = hc4->chainTable; + U32* const HashTable = hc4->hashTable; + const BYTE* const base = hc4->base; + const U32 dictLimit = hc4->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1); + const BYTE* const dictBase = hc4->dictBase; + U32 matchIndex; + int nbAttempts = maxNbAttempts; + int delta = (int)(ip-iLowLimit); + + + /* First Match */ + LZ4HC_Insert(hc4, ip); + matchIndex = HashTable[LZ4HC_hashPtr(ip)]; + + while ((matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + const BYTE* matchPtr = base + matchIndex; + if (*(iLowLimit + longest) == *(matchPtr - delta + longest)) + if (LZ4_read32(matchPtr) == LZ4_read32(ip)) + { + int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit); + int back = 0; + + while ((ip+back>iLowLimit) + && (matchPtr+back > lowPrefixPtr) + && (ip[back-1] == matchPtr[back-1])) + back--; + + mlt -= back; + + if (mlt > longest) + { + longest = (int)mlt; + *matchpos = matchPtr+back; + *startpos = ip+back; + } + } + } + else + { + const BYTE* matchPtr = dictBase + matchIndex; + if (LZ4_read32(matchPtr) == LZ4_read32(ip)) + { + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--; + mlt -= back; + if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } + } + } + matchIndex -= DELTANEXTU16(matchIndex); + } + + return longest; +} + + +typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; + +#define LZ4HC_DEBUG 0 +#if LZ4HC_DEBUG +static unsigned debug = 0; +#endif + +FORCE_INLINE int LZ4HC_encodeSequence ( + const BYTE** ip, + BYTE** op, + const BYTE** anchor, + int matchLength, + const BYTE* const match, + limitedOutput_directive limitedOutputBuffer, + BYTE* oend) +{ + int length; + BYTE* token; + +#if LZ4HC_DEBUG + if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match)); +#endif + + /* Encode Literal length */ + length = (int)(*ip - *anchor); + token = (*op)++; + if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } + else *token = (BYTE)(length<>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } + else *token += (BYTE)(length); + + /* Prepare next loop */ + *ip += matchLength; + *anchor = *ip; + + return 0; +} + + +static int LZ4HC_compress_generic ( + void* ctxvoid, + const char* source, + char* dest, + int inputSize, + int maxOutputSize, + int compressionLevel, + limitedOutput_directive limit + ) +{ + LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid; + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + unsigned maxNbAttempts; + int ml, ml2, ml3, ml0; + const BYTE* ref=NULL; + const BYTE* start2=NULL; + const BYTE* ref2=NULL; + const BYTE* start3=NULL; + const BYTE* ref3=NULL; + const BYTE* start0; + const BYTE* ref0; + + + /* init */ + if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel; + if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default; + maxNbAttempts = 1 << (compressionLevel-1); + ctx->end += inputSize; + + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts); + if (!ml) { ip++; continue; } + + /* saved, in case we would skip too much */ + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search2: + if (ip+ml < mflimit) + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts); + else ml2 = ml; + + if (ml2 == ml) /* No better match */ + { + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + continue; + } + + if (start0 < ip) + { + if (start2 < ip + ml0) /* empirical */ + { + ip = start0; + ref = ref0; + ml = ml0; + } + } + + /* Here, start0==ip */ + if ((start2 - ip) < 3) /* First Match too small : removed */ + { + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + /* + * Currently we have : + * ml2 > ml1, and + * ip1+3 <= ip2 (usually < ip1+ml1) + */ + if ((start2 - ip) < OPTIMAL_ML) + { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */ + + if (start2 + ml2 < mflimit) + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts); + else ml3 = ml2; + + if (ml3 == ml2) /* No better match : 2 sequences to encode */ + { + /* ip & ref are known; Now for ml */ + if (start2 < ip+ml) ml = (int)(start2 - ip); + /* Now, encode 2 sequences */ + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + ip = start2; + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0; + continue; + } + + if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */ + { + if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */ + { + if (start2 < ip+ml) + { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < MINMATCH) + { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + /* + * OK, now we have 3 ascending matches; let's write at least the first one + * ip & ref are known; Now for ml + */ + if (start2 < ip+ml) + { + if ((start2 - ip) < (int)ML_MASK) + { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + else + { + ml = (int)(start2 - ip); + } + } + if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0; + + ip = start2; + ref = ref2; + ml = ml2; + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + + goto _Search3; + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<base = NULL; + ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel; +} + +int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize) +{ + LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr; + if (dictSize > 64 KB) + { + dictionary += dictSize - 64 KB; + dictSize = 64 KB; + } + LZ4HC_init (ctxPtr, (const BYTE*)dictionary); + if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + return dictSize; +} + + +/* compression */ + +static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ +} + +static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize, limitedOutput_directive limit) +{ + /* auto-init if forgotten */ + if (ctxPtr->base == NULL) + LZ4HC_init (ctxPtr, (const BYTE*) source); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) + { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; + if (dictSize > 64 KB) dictSize = 64 KB; + + LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)source != ctxPtr->end) + LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; + if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) + { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; + } + } + + return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit); +} + +int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + if (maxOutputSize < LZ4_compressBound(inputSize)) + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + else + return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); +} + + +/* dictionary saving */ + +int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr; + int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + if (dictSize > 64 KB) dictSize = 64 KB; + if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + { + U32 endIndex = (U32)(streamPtr->end - streamPtr->base); + streamPtr->end = (const BYTE*)safeBuffer + dictSize; + streamPtr->base = streamPtr->end - endIndex; + streamPtr->dictLimit = endIndex - dictSize; + streamPtr->lowLimit = endIndex - dictSize; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + } + return dictSize; +} + + +/*********************************** +* Deprecated Functions +***********************************/ +/* Deprecated compression functions */ +/* These functions are planned to start generate warnings by r131 approximately */ +int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); } +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); } +int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); } +int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); } +int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); } +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } + + +/* Deprecated streaming functions */ +/* These functions currently generate deprecation warnings */ +int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; } + +int LZ4_resetStreamStateHC(void* state, char* inputBuffer) +{ + if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */ + LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer); + ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer; + return 0; +} + +void* LZ4_createHC (char* inputBuffer) +{ + void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure)); + if (hc4 == NULL) return NULL; /* not enough memory */ + LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer); + ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer; + return hc4; +} + +int LZ4_freeHC (void* LZ4HC_Data) +{ + FREEMEM(LZ4HC_Data); + return (0); +} + +int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit); +} + +int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel) +{ + return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput); +} + +char* LZ4_slideInputBufferHC(void* LZ4HC_Data) +{ + LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data; + int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB); + return (char*)(hc4->inputBuffer + dictSize); +} diff --git a/C/lz4/lz4hc.h b/C/lz4/lz4hc.h new file mode 100644 index 00000000..431f7c87 --- /dev/null +++ b/C/lz4/lz4hc.h @@ -0,0 +1,189 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Header File + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 source repository : https://github.com/Cyan4973/lz4 + - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c +*/ +#pragma once + + +#if defined (__cplusplus) +extern "C" { +#endif + +/***************************** +* Includes +*****************************/ +#include /* size_t */ + + +/************************************** +* Block Compression +**************************************/ +int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); +/* +LZ4_compress_HC : + Destination buffer 'dst' must be already allocated. + Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible) + Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h") + srcSize : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h") + compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work. + 0 means "use default value" (see lz4hc.c). + Values >16 behave the same as 16. + return : the number of bytes written into buffer 'dst' + or 0 if compression fails. +*/ + + +/* Note : + Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license) +*/ + + +int LZ4_sizeofStateHC(void); +int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); +/* +LZ4_compress_HC_extStateHC() : + Use this function if you prefer to manually allocate memory for compression tables. + To know how much memory must be allocated for the compression tables, use : + int LZ4_sizeofStateHC(); + + Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). + + The allocated memory can then be provided to the compression functions using 'void* state' parameter. + LZ4_compress_HC_extStateHC() is equivalent to previously described function. + It just uses externally allocated memory for stateHC. +*/ + + +/************************************** +* Streaming Compression +**************************************/ +#define LZ4_STREAMHCSIZE 262192 +#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t)) +typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t; +/* + LZ4_streamHC_t + This structure allows static allocation of LZ4 HC streaming state. + State must then be initialized using LZ4_resetStreamHC() before first use. + + Static allocation should only be used in combination with static linking. + If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof. +*/ + + +LZ4_streamHC_t* LZ4_createStreamHC(void); +int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr); +/* + These functions create and release memory for LZ4 HC streaming state. + Newly created states are already initialized. + Existing state space can be re-used anytime using LZ4_resetStreamHC(). + If you use LZ4 as a DLL, use these functions instead of static structure allocation, + to avoid size mismatch between different versions. +*/ + +void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel); +int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); + +int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + +/* + These functions compress data in successive blocks of any size, using previous blocks as dictionary. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller 64 KB. + Such case is automatically detected and correctly handled by LZ4_compress_HC_continue(). + + Before starting compression, state must be properly initialized, using LZ4_resetStreamHC(). + A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional). + + Then, use LZ4_compress_HC_continue() to compress each successive block. + It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression. + Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. + As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation. + + If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, + you must save it to a safer memory space, using LZ4_saveDictHC(). + Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. +*/ + + + +/************************************** +* Deprecated Functions +**************************************/ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */ +#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_DEPRECATE_WARNING_DEFBLOCK +# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +# if (LZ4_GCC_VERSION >= 405) || defined(__clang__) +# define LZ4_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ4_GCC_VERSION >= 301) +# define LZ4_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ4_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler") +# define LZ4_DEPRECATED(message) +# endif +#endif // LZ4_DEPRECATE_WARNING_DEFBLOCK + +/* compression functions */ +/* these functions are planned to trigger warning messages by r131 approximately */ +int LZ4_compressHC (const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel); +int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize); +int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Streaming functions following the older model; should no longer be used */ +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer); +LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel); +LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void); +LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer); + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz4/lz4xxhash.c b/C/lz4/lz4xxhash.c new file mode 100644 index 00000000..e6fb8f14 --- /dev/null +++ b/C/lz4/lz4xxhash.c @@ -0,0 +1,915 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2015, Yann Collet + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/************************************** +* Tuning parameters +**************************************/ +/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. + * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). + */ +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +/* XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/* XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1. + * It will improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#define XXH_FORCE_NATIVE_FORMAT 0 + + +/************************************** +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/************************************** +* Includes & Memory related functions +***************************************/ +#include "xxhash.h" +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + + +/************************************** +* Basic Types +***************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +static U32 XXH_read32(const void* memPtr) +{ + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} + + + +/****************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/*************************************** +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ +static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) +#endif + + +/***************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + + +/*************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/*************************************** +* Constants +***************************************/ +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + + +/***************************** +* Simple Hash Functions +*****************************/ +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +unsigned XXH32 (const void* input, size_t len, unsigned seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } + else + { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/**************************************************** +* Advanced Hash Functions +****************************************************/ + +/*** Allocation ***/ +typedef struct +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + U32 mem32[4]; /* defined as U32 for alignment */ + U32 memsize; +} XXH_istate32_t; + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + U64 mem64[4]; /* defined as U64 for alignment */ + U32 memsize; +} XXH_istate64_t; + + +XXH32_state_t* XXH32_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH64_state_t* XXH64_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) +{ + XXH_istate64_t* state = (XXH_istate64_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) /* fill in tmp buffer */ + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) /* some data left from previous update */ + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { + const U32* p32 = state->mem32; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v1 = XXH_rotl32(state->v1, 13); + state->v1 *= PRIME32_1; + p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v2 = XXH_rotl32(state->v2, 13); + state->v2 *= PRIME32_1; + p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v3 = XXH_rotl32(state->v3, 13); + state->v3 *= PRIME32_1; + p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v4 = XXH_rotl32(state->v4, 13); + state->v4 *= PRIME32_1; + p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32(p, endian) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_readLE32(p, endian) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_readLE32(p, endian) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_readLE32(p, endian) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem32, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) +{ + const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p+4<=bEnd) + { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) /* fill in tmp buffer */ + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) /* some data left from previous update */ + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + { + const U64* p64 = state->mem64; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64(p, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + p+=8; + v2 += XXH_readLE64(p, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + p+=8; + v3 += XXH_readLE64(p, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + p+=8; + v4 += XXH_readLE64(p, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + p+=8; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem64, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) +{ + const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } + else + { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_readLE64(p, endian); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + diff --git a/C/lz4/xxhash.h b/C/lz4/xxhash.h new file mode 100644 index 00000000..c60aa615 --- /dev/null +++ b/C/lz4/xxhash.h @@ -0,0 +1,192 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +/***************************** +* Definitions +*****************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/***************************** +* Namespace Emulation +*****************************/ +/* Motivations : + +If you need to include xxHash into your library, +but wish to avoid xxHash symbols to be present on your library interface +in an effort to avoid potential name collision if another library also includes xxHash, + +you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash +with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values). + +Note that no change is required within the calling program : +it can still call xxHash functions using their regular name. +They will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#endif + + +/***************************** +* Simple Hash Functions +*****************************/ + +unsigned int XXH32 (const void* input, size_t length, unsigned seed); +unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Faster on 64-bits systems. Slower on 32-bits systems. +*/ + + + +/***************************** +* Advanced Hash Functions +*****************************/ +typedef struct { long long ll[ 6]; } XXH32_state_t; +typedef struct { long long ll[11]; } XXH64_state_t; + +/* +These structures allow static allocation of XXH states. +States must then be initialized using XXHnn_reset() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +XXH32_state_t* XXH32_createState(void); +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH64_state_t* XXH64_createState(void); +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/* +These functions create and release memory for XXH state. +States must then be initialized using XXHnn_reset() before first use. +*/ + + +XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); +XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +unsigned long long XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions calculate the xxHash of an input provided in multiple smaller packets, +as opposed to an input provided as a single block. + +XXH state space must first be allocated, using either static or dynamic method provided above. + +Start a new hash by initializing state with a seed, using XXHnn_reset(). + +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, meaning allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, you can produce a hash anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You can nonetheless continue feeding the hash state with more input, +and therefore get some new hashes, by calling again XXHnn_digest(). + +When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). +*/ + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz5/lz5.c b/C/lz5/lz5.c new file mode 100644 index 00000000..dfd75624 --- /dev/null +++ b/C/lz5/lz5.c @@ -0,0 +1,1355 @@ +/* + LZ5 - Fast LZ compression algorithm + Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2015, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ5 source repository : https://github.com/inikep/lz5 + - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ + + + +/************************************** +* Includes +**************************************/ +#include "lz5common.h" +#include "lz5.h" +#include + + +/************************************** +* Local Constants +**************************************/ +#define LZ5_HASHLOG (LZ5_MEMORY_USAGE-2) +#define HASH_SIZE_U32 (1 << LZ5_HASHLOG) /* required as macro for static allocation */ + +static const int LZ5_64Klimit = ((64 KB) + (MFLIMIT-1)); +static const U32 LZ5_skipTrigger = 6; /* Increase this value ==> compression run slower on incompressible data */ + + +/************************************** +* Local Structures and types +**************************************/ +typedef struct { + U32 hashTable[HASH_SIZE_U32]; + U32 currentOffset; + U32 initCheck; + const BYTE* dictionary; + BYTE* bufferStart; /* obsolete, used for slideInputBuffer */ + U32 dictSize; +} LZ5_stream_t_internal; + +typedef enum { notLimited = 0, limitedOutput = 1 } limitedOutput_directive; +typedef enum { byPtr, byU32, byU16 } tableType_t; + +typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; +typedef enum { noDictIssue = 0, dictSmall } dictIssue_directive; + +typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; +typedef enum { full = 0, partial = 1 } earlyEnd_directive; + + +/************************************** +* Local Utils +**************************************/ +int LZ5_versionNumber (void) { return LZ5_VERSION_NUMBER; } +int LZ5_compressBound(int isize) { return LZ5_COMPRESSBOUND(isize); } +int LZ5_sizeofState() { return LZ5_STREAMSIZE; } + + + +/******************************** +* Compression functions +********************************/ + +static U32 LZ5_hashSequence(U32 sequence, tableType_t const tableType) +{ + if (tableType == byU16) + return (((sequence) * prime4bytes) >> ((32)-(LZ5_HASHLOG+1))); + else + return (((sequence) * prime4bytes) >> ((32)-LZ5_HASHLOG)); +} + +static U32 LZ5_hashSequence64(size_t sequence, tableType_t const tableType) +{ + const U32 hashLog = (tableType == byU16) ? LZ5_HASHLOG+1 : LZ5_HASHLOG; + const U32 hashMask = (1<> (40 - hashLog)) & hashMask; +} + +static U32 LZ5_hashSequenceT(size_t sequence, tableType_t const tableType) +{ + if (MEM_64bits()) + return LZ5_hashSequence64(sequence, tableType); + return LZ5_hashSequence((U32)sequence, tableType); +} + +static U32 LZ5_hashPosition(const void* p, tableType_t tableType) { return LZ5_hashSequenceT(MEM_read_ARCH(p), tableType); } + +static void LZ5_putPositionOnHash(const BYTE* p, U32 h, void* tableBase, tableType_t const tableType, const BYTE* srcBase) +{ + switch (tableType) + { + case byPtr: { const BYTE** hashTable = (const BYTE**)tableBase; hashTable[h] = p; return; } + case byU32: { U32* hashTable = (U32*) tableBase; hashTable[h] = (U32)(p-srcBase); return; } + case byU16: { U16* hashTable = (U16*) tableBase; hashTable[h] = (U16)(p-srcBase); return; } + } +} + +static void LZ5_putPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ5_hashPosition(p, tableType); + LZ5_putPositionOnHash(p, h, tableBase, tableType, srcBase); +} + +static const BYTE* LZ5_getPositionOnHash(U32 h, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + if (tableType == byPtr) { const BYTE** hashTable = (const BYTE**) tableBase; return hashTable[h]; } + if (tableType == byU32) { U32* hashTable = (U32*) tableBase; return hashTable[h] + srcBase; } + { U16* hashTable = (U16*) tableBase; return hashTable[h] + srcBase; } /* default, to ensure a return */ +} + +static const BYTE* LZ5_getPosition(const BYTE* p, void* tableBase, tableType_t tableType, const BYTE* srcBase) +{ + U32 h = LZ5_hashPosition(p, tableType); + return LZ5_getPositionOnHash(h, tableBase, tableType, srcBase); +} + +FORCE_INLINE int LZ5_compress_generic( + void* const ctx, + const char* const source, + char* const dest, + const int inputSize, + const int maxOutputSize, + const limitedOutput_directive outputLimited, + const tableType_t tableType, + const dict_directive dict, + const dictIssue_directive dictIssue, + const U32 acceleration) +{ + LZ5_stream_t_internal* const dictPtr = (LZ5_stream_t_internal*)ctx; + + const BYTE* ip = (const BYTE*) source; + const BYTE* base; + const BYTE* lowLimit; + const BYTE* const lowRefLimit = ip - dictPtr->dictSize; + const BYTE* const dictionary = dictPtr->dictionary; + const BYTE* const dictEnd = dictionary + dictPtr->dictSize; + const size_t dictDelta = dictEnd - (const BYTE*)source; + const BYTE* anchor = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dest; + BYTE* const olimit = op + maxOutputSize; + + U32 forwardH, last_off=1; + size_t refDelta=0; + + /* Init conditions */ + if ((U32)inputSize > (U32)LZ5_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + switch(dict) + { + case noDict: + default: + base = (const BYTE*)source; + lowLimit = (const BYTE*)source; + break; + case withPrefix64k: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source - dictPtr->dictSize; + break; + case usingExtDict: + base = (const BYTE*)source - dictPtr->currentOffset; + lowLimit = (const BYTE*)source; + break; + } + if ((tableType == byU16) && (inputSize>=LZ5_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (inputSize> LZ5_skipTrigger); + + if (unlikely(forwardIp > mflimit)) goto _last_literals; + + match = LZ5_getPositionOnHash(h, ctx, tableType, base); + if (dict==usingExtDict) + { + if (match<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + forwardH = LZ5_hashPosition(forwardIp, tableType); + LZ5_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((dictIssue==dictSmall) ? (match < lowRefLimit) : 0) + || ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (MEM_read32(match+refDelta) != MEM_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match+refDelta > lowLimit) && (unlikely(ip[-1]==match[refDelta-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if ((outputLimited) && (unlikely(op + litLength + (2 + 1 + LASTLITERALS) + (litLength/255) > olimit))) + return 0; /* Check output limit */ + + if (ip-match >= LZ5_SHORT_OFFSET_DISTANCE && ip-match < LZ5_MID_OFFSET_DISTANCE && (U32)(ip-match) != last_off) + { + if (litLength>=RUN_MASK) + { + int len = (int)litLength-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<=RUN_MASK2) + { + int len = (int)litLength-RUN_MASK2; + *token=(RUN_MASK2<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<>8))< matchlimit) limit = matchlimit; + matchLength = MEM_count(ip+MINMATCH, match+MINMATCH, limit); + ip += MINMATCH + matchLength; + if (ip==limit) + { + size_t more = MEM_count(ip, (const BYTE*)source, matchlimit); + matchLength += more; + ip += more; + } + } + else + { + matchLength = MEM_count(ip+MINMATCH, match+MINMATCH, matchlimit); + ip += MINMATCH + matchLength; + } + + if ((outputLimited) && (unlikely(op + (1 + LASTLITERALS) + (matchLength>>8) > olimit))) + return 0; /* Check output limit */ + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + for (; matchLength >= 510 ; matchLength-=510) { *op++ = 255; *op++ = 255; } + if (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of chunk */ + if (ip > mflimit) break; + + /* Fill table */ + LZ5_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ5_getPosition(ip, ctx, tableType, base); + if (dict==usingExtDict) + { + if (match<(const BYTE*)source) + { + refDelta = dictDelta; + lowLimit = dictionary; + } + else + { + refDelta = 0; + lowLimit = (const BYTE*)source; + } + } + LZ5_putPosition(ip, ctx, tableType, base); + if ( ((dictIssue==dictSmall) ? (match>=lowRefLimit) : 1) + && (match+MAX_DISTANCE>=ip) + && (MEM_read32(match+refDelta)==MEM_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ5_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + const size_t lastRun = (size_t)(iend - anchor); + if ((outputLimited) && ((op - (BYTE*)dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) + return 0; /* Check output limit */ + if (lastRun >= RUN_MASK) + { + size_t accumulator = lastRun - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRun<= LZ5_compressBound(inputSize)) + { + if (inputSize < LZ5_64Klimit) + return LZ5_compress_generic(state, source, dest, inputSize, 0, notLimited, byU16, noDict, noDictIssue, acceleration); + else + return LZ5_compress_generic(state, source, dest, inputSize, 0, notLimited, MEM_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } + else + { + if (inputSize < LZ5_64Klimit) + return LZ5_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ5_compress_generic(state, source, dest, inputSize, maxOutputSize, limitedOutput, MEM_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); + } +} + + +int LZ5_compress_fast(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ +#if (HEAPMODE) + void* ctxPtr = ALLOCATOR(1, sizeof(LZ5_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ5_stream_t ctx; + void* ctxPtr = &ctx; +#endif + + int result = LZ5_compress_fast_extState(ctxPtr, source, dest, inputSize, maxOutputSize, acceleration); + +#if (HEAPMODE) + FREEMEM(ctxPtr); +#endif + return result; +} + + +int LZ5_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) +{ + return LZ5_compress_fast(source, dest, inputSize, maxOutputSize, 1); +} + + +/* hidden debug function */ +/* strangely enough, gcc generates faster code when this function is uncommented, even if unused */ +int LZ5_compress_fast_force(const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ5_stream_t ctx; + + LZ5_resetStream(&ctx); + + if (inputSize < LZ5_64Klimit) + return LZ5_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, byU16, noDict, noDictIssue, acceleration); + else + return LZ5_compress_generic(&ctx, source, dest, inputSize, maxOutputSize, limitedOutput, MEM_64bits() ? byU32 : byPtr, noDict, noDictIssue, acceleration); +} + + +/******************************** +* destSize variant +********************************/ + +static int LZ5_compress_destSize_generic( + void* const ctx, + const char* const src, + char* const dst, + int* const srcSizePtr, + const int targetDstSize, + const tableType_t tableType) +{ + const BYTE* ip = (const BYTE*) src; + const BYTE* base = (const BYTE*) src; + const BYTE* lowLimit = (const BYTE*) src; + const BYTE* anchor = ip; + const BYTE* const iend = ip + *srcSizePtr; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = iend - LASTLITERALS; + + BYTE* op = (BYTE*) dst; + BYTE* const oend = op + targetDstSize; + BYTE* const oMaxLit = op + targetDstSize - 2 /* offset */ - 8 /* because 8+MINMATCH==MFLIMIT */ - 1 /* token */; + BYTE* const oMaxMatch = op + targetDstSize - (LASTLITERALS + 1 /* token */); + BYTE* const oMaxSeq = oMaxLit - 1 /* token */; + + U32 forwardH, last_off=1; + + + /* Init conditions */ + if (targetDstSize < 1) return 0; /* Impossible to store anything */ + if ((U32)*srcSizePtr > (U32)LZ5_MAX_INPUT_SIZE) return 0; /* Unsupported input size, too large (or negative) */ + if ((tableType == byU16) && (*srcSizePtr>=LZ5_64Klimit)) return 0; /* Size too large (not within 64K limit) */ + if (*srcSizePtr> LZ5_skipTrigger); + + if (unlikely(forwardIp > mflimit)) + goto _last_literals; + + match = LZ5_getPositionOnHash(h, ctx, tableType, base); + forwardH = LZ5_hashPosition(forwardIp, tableType); + LZ5_putPositionOnHash(ip, h, ctx, tableType, base); + + } while ( ((tableType==byU16) ? 0 : (match + MAX_DISTANCE < ip)) + || (MEM_read32(match) != MEM_read32(ip)) ); + } + + /* Catch up */ + while ((ip>anchor) && (match > lowLimit) && (unlikely(ip[-1]==match[-1]))) { ip--; match--; } + + { + /* Encode Literal length */ + unsigned litLength = (unsigned)(ip - anchor); + token = op++; + if (op + ((litLength+240)/255) + litLength > oMaxLit) + { + /* Not enough space for a last match */ + op--; + goto _last_literals; + } + + if ((U32)(ip-match) >= LZ5_SHORT_OFFSET_DISTANCE && (U32)(ip-match) < LZ5_MID_OFFSET_DISTANCE && (U32)(ip-match) != last_off) + { + if (litLength>=RUN_MASK) + { + int len = (int)litLength-RUN_MASK; + *token=(RUN_MASK<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<=RUN_MASK2) + { + int len = (int)litLength-RUN_MASK2; + *token=(RUN_MASK2<= 255 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (BYTE)(litLength<>8))< oMaxMatch) + { + /* Match description too long : reduce it */ + matchLength = (15-1) + (oMaxMatch-op) * 255; + } + ip += MINMATCH + matchLength; + + if (matchLength>=ML_MASK) + { + *token += ML_MASK; + matchLength -= ML_MASK; + while (matchLength >= 255) { matchLength-=255; *op++ = 255; } + *op++ = (BYTE)matchLength; + } + else *token += (BYTE)(matchLength); + } + + anchor = ip; + + /* Test end of block */ + if (ip > mflimit) break; + if (op > oMaxSeq) break; + + /* Fill table */ + LZ5_putPosition(ip-2, ctx, tableType, base); + + /* Test next position */ + match = LZ5_getPosition(ip, ctx, tableType, base); + LZ5_putPosition(ip, ctx, tableType, base); + if ( (match+MAX_DISTANCE>=ip) + && (MEM_read32(match)==MEM_read32(ip)) ) + { token=op++; *token=0; goto _next_match; } + + /* Prepare next loop */ + forwardH = LZ5_hashPosition(++ip, tableType); + } + +_last_literals: + /* Encode Last Literals */ + { + size_t lastRunSize = (size_t)(iend - anchor); + if (op + 1 /* token */ + ((lastRunSize+240)/255) /* litLength */ + lastRunSize /* literals */ > oend) + { + /* adapt lastRunSize to fill 'dst' */ + lastRunSize = (oend-op) - 1; + lastRunSize -= (lastRunSize+240)/255; + } + ip = anchor + lastRunSize; + + if (lastRunSize >= RUN_MASK) + { + size_t accumulator = lastRunSize - RUN_MASK; + *op++ = RUN_MASK << ML_BITS; + for(; accumulator >= 255 ; accumulator-=255) *op++ = 255; + *op++ = (BYTE) accumulator; + } + else + { + *op++ = (BYTE)(lastRunSize<= LZ5_compressBound(*srcSizePtr)) /* compression success is guaranteed */ + { + return LZ5_compress_fast_extState(state, src, dst, *srcSizePtr, targetDstSize, 1); + } + else + { + if (*srcSizePtr < LZ5_64Klimit) + return LZ5_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, byU16); + else + return LZ5_compress_destSize_generic(state, src, dst, srcSizePtr, targetDstSize, MEM_64bits() ? byU32 : byPtr); + } +} + + +int LZ5_compress_destSize(const char* src, char* dst, int* srcSizePtr, int targetDstSize) +{ +#if (HEAPMODE) + void* ctx = ALLOCATOR(1, sizeof(LZ5_stream_t)); /* malloc-calloc always properly aligned */ +#else + LZ5_stream_t ctxBody; + void* ctx = &ctxBody; +#endif + + int result = LZ5_compress_destSize_extState(ctx, src, dst, srcSizePtr, targetDstSize); + +#if (HEAPMODE) + FREEMEM(ctx); +#endif + return result; +} + + + +/******************************** +* Streaming functions +********************************/ + +LZ5_stream_t* LZ5_createStream(void) +{ + LZ5_stream_t* lz5s = (LZ5_stream_t*)ALLOCATOR(8, LZ5_STREAMSIZE_U64); + LZ5_STATIC_ASSERT(LZ5_STREAMSIZE >= sizeof(LZ5_stream_t_internal)); /* A compilation error here means LZ5_STREAMSIZE is not large enough */ + LZ5_resetStream(lz5s); + return lz5s; +} + +void LZ5_resetStream (LZ5_stream_t* LZ5_stream) +{ + MEM_INIT(LZ5_stream, 0, sizeof(LZ5_stream_t)); +} + +int LZ5_freeStream (LZ5_stream_t* LZ5_stream) +{ + FREEMEM(LZ5_stream); + return (0); +} + + +#define HASH_UNIT sizeof(size_t) +int LZ5_loadDict (LZ5_stream_t* LZ5_dict, const char* dictionary, int dictSize) +{ + LZ5_stream_t_internal* dict = (LZ5_stream_t_internal*) LZ5_dict; + const BYTE* p = (const BYTE*)dictionary; + const BYTE* const dictEnd = p + dictSize; + const BYTE* base; + + if ((dict->initCheck) || (dict->currentOffset > 1 GB)) /* Uninitialized structure, or reuse overflow */ + LZ5_resetStream(LZ5_dict); + + /* if (dictSize < (int)HASH_UNIT) + { + dict->dictionary = NULL; + dict->dictSize = 0; + return 0; + }*/ + + if ((dictEnd - p) > LZ5_DICT_SIZE) p = dictEnd - LZ5_DICT_SIZE; + dict->currentOffset += LZ5_DICT_SIZE; + base = p - dict->currentOffset; + dict->dictionary = p; + dict->dictSize = (U32)(dictEnd - p); + dict->currentOffset += dict->dictSize; + + while (p <= dictEnd-HASH_UNIT) + { + LZ5_putPosition(p, dict->hashTable, byU32, base); + p+=3; + } + + return dict->dictSize; +} + + +static void LZ5_renormDictT(LZ5_stream_t_internal* LZ5_dict, const BYTE* src) +{ + if ((LZ5_dict->currentOffset > 0x80000000) || + ((size_t)LZ5_dict->currentOffset > (size_t)src)) /* address space overflow */ + { + /* rescale hash table */ + U32 delta = LZ5_dict->currentOffset - LZ5_DICT_SIZE; + const BYTE* dictEnd = LZ5_dict->dictionary + LZ5_dict->dictSize; + int i; + for (i=0; ihashTable[i] < delta) LZ5_dict->hashTable[i]=0; + else LZ5_dict->hashTable[i] -= delta; + } + LZ5_dict->currentOffset = LZ5_DICT_SIZE; + if (LZ5_dict->dictSize > LZ5_DICT_SIZE) LZ5_dict->dictSize = LZ5_DICT_SIZE; + LZ5_dict->dictionary = dictEnd - LZ5_dict->dictSize; + } +} + + +int LZ5_compress_fast_continue (LZ5_stream_t* LZ5_stream, const char* source, char* dest, int inputSize, int maxOutputSize, int acceleration) +{ + LZ5_stream_t_internal* streamPtr = (LZ5_stream_t_internal*)LZ5_stream; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = (const BYTE*) source; + if (streamPtr->initCheck) return 0; /* Uninitialized structure detected */ + if ((streamPtr->dictSize>0) && (smallest>dictEnd)) smallest = dictEnd; + LZ5_renormDictT(streamPtr, smallest); + if (acceleration < 1) acceleration = ACCELERATION_DEFAULT; + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + if ((sourceEnd > streamPtr->dictionary) && (sourceEnd < dictEnd)) + { + streamPtr->dictSize = (U32)(dictEnd - sourceEnd); + if (streamPtr->dictSize > LZ5_DICT_SIZE) streamPtr->dictSize = LZ5_DICT_SIZE; + if (streamPtr->dictSize < 4) streamPtr->dictSize = 0; + streamPtr->dictionary = dictEnd - streamPtr->dictSize; + } + } + + /* prefix mode : source data follows dictionary */ + if (dictEnd == (const BYTE*)source) + { + int result; + if ((streamPtr->dictSize < LZ5_DICT_SIZE) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, dictSmall, acceleration); + else + result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, withPrefix64k, noDictIssue, acceleration); + streamPtr->dictSize += (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } + + /* external dictionary mode */ + { + int result; + if ((streamPtr->dictSize < LZ5_DICT_SIZE) && (streamPtr->dictSize < streamPtr->currentOffset)) + result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, dictSmall, acceleration); + else + result = LZ5_compress_generic(LZ5_stream, source, dest, inputSize, maxOutputSize, limitedOutput, byU32, usingExtDict, noDictIssue, acceleration); + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + return result; + } +} + + +/* Hidden debug function, to force external dictionary mode */ +int LZ5_compress_forceExtDict (LZ5_stream_t* LZ5_dict, const char* source, char* dest, int inputSize) +{ + LZ5_stream_t_internal* streamPtr = (LZ5_stream_t_internal*)LZ5_dict; + int result; + const BYTE* const dictEnd = streamPtr->dictionary + streamPtr->dictSize; + + const BYTE* smallest = dictEnd; + if (smallest > (const BYTE*) source) smallest = (const BYTE*) source; + LZ5_renormDictT((LZ5_stream_t_internal*)LZ5_dict, smallest); + + result = LZ5_compress_generic(LZ5_dict, source, dest, inputSize, 0, notLimited, byU32, usingExtDict, noDictIssue, 1); + + streamPtr->dictionary = (const BYTE*)source; + streamPtr->dictSize = (U32)inputSize; + streamPtr->currentOffset += (U32)inputSize; + + return result; +} + + +int LZ5_saveDict (LZ5_stream_t* LZ5_dict, char* safeBuffer, int dictSize) +{ + LZ5_stream_t_internal* dict = (LZ5_stream_t_internal*) LZ5_dict; + const BYTE* previousDictEnd = dict->dictionary + dict->dictSize; + if (!dict->dictionary) + return 0; + + if ((U32)dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; /* useless to define a dictionary > LZ5_DICT_SIZE */ + if ((U32)dictSize > dict->dictSize) dictSize = dict->dictSize; + + memmove(safeBuffer, previousDictEnd - dictSize, dictSize); + + dict->dictionary = (const BYTE*)safeBuffer; + dict->dictSize = (U32)dictSize; + + return dictSize; +} + + + +/******************************* +* Decompression functions +*******************************/ +/* + * This generic decompression function cover all use cases. + * It shall be instantiated several times, using different sets of directives + * Note that it is essential this generic function is really inlined, + * in order to remove useless branches during compilation optimization. + */ +FORCE_INLINE int LZ5_decompress_generic( + const char* const source, + char* const dest, + int inputSize, + int outputSize, /* If endOnInput==endOnInputSize, this value is the max size of Output Buffer. */ + + int endOnInput, /* endOnOutputSize, endOnInputSize */ + int partialDecoding, /* full, partial */ + int targetOutputSize, /* only used if partialDecoding==partial */ + int dict, /* noDict, withPrefix64k, usingExtDict */ + const BYTE* const lowPrefix, /* == dest if dict == noDict */ + const BYTE* const dictStart, /* only if dict==usingExtDict */ + const size_t dictSize /* note : = 0 if noDict */ + ) +{ + /* Local Variables */ + const BYTE* ip = (const BYTE*) source; + const BYTE* const iend = ip + inputSize; + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + outputSize; + BYTE* cpy; + BYTE* oexit = op + targetOutputSize; + const BYTE* const lowLimit = lowPrefix - dictSize; + + const BYTE* const dictEnd = (const BYTE*)dictStart + dictSize; + const unsigned dec32table[] = {4, 1, 2, 1, 4, 4, 4, 4}; + const int dec64table[] = {0, 0, 0, -1, 0, 1, 2, 3}; + + const int safeDecode = (endOnInput==endOnInputSize); + const int checkOffset = ((safeDecode) && (dictSize < (int)(LZ5_DICT_SIZE))); + + size_t last_off = 1; + + /* Special cases */ + if ((partialDecoding) && (oexit> oend-MFLIMIT)) oexit = oend-MFLIMIT; /* targetOutputSize too high => decode everything */ + if ((endOnInput) && (unlikely(outputSize==0))) return ((inputSize==1) && (*ip==0)) ? 0 : -1; /* Empty output buffer */ + if ((!endOnInput) && (unlikely(outputSize==0))) return (*ip==0?1:-1); + + + /* Main Loop */ + while (1) + { + unsigned token; + size_t length; + const BYTE* match; + size_t offset; + + /* get literal length */ + token = *ip++; + if (token>>6) + { + if ((length=(token>>ML_BITS)&RUN_MASK2) == RUN_MASK2) + { + unsigned s; + do + { + s = *ip++; + length += s; + } + while (likely((endOnInput)?ip>ML_BITS)&RUN_MASK) == RUN_MASK) + { + unsigned s; + do + { + s = *ip++; + length += s; + } + while (likely((endOnInput)?ip(partialDecoding?oexit:oend-WILDCOPYLENGTH)) || (ip+length>iend-(0+1+LASTLITERALS))) ) + || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH))) + { + if (partialDecoding) + { + if (cpy > oend) goto _output_error; /* Error : write attempt beyond end of output buffer */ + if ((endOnInput) && (ip+length > iend)) goto _output_error; /* Error : read attempt beyond end of input buffer */ + } + else + { + if ((!endOnInput) && (cpy != oend)) goto _output_error; /* Error : block decoding must stop exactly there */ + if ((endOnInput) && ((ip+length != iend) || (cpy > oend))) goto _output_error; /* Error : input must be consumed */ + } + memcpy(op, ip, length); + ip += length; + op += length; + break; /* Necessarily EOF, due to parsing restrictions */ + } + MEM_wildCopy(op, ip, cpy); + ip += length; op = cpy; + + /* get offset */ +#if 0 + switch (token>>6) + { + default: offset = *ip + (((token>>ML_RUN_BITS2)&3)<<8); ip++; break; + case 0: offset = MEM_readLE16(ip); ip+=2; break; + case 1: + if ((token>>5) == 3) + offset = last_off; + else // (token>>ML_RUN_BITS2) == 2 + { offset = MEM_readLE24(ip); ip+=3; } + break; + } +#else + if (token>>7) + { + offset = *ip + (((token>>ML_RUN_BITS2)&3)<<8); ip++; + } + else + if ((token>>ML_RUN_BITS) == 0) + { + offset = MEM_readLE16(ip); ip+=2; + } + else + if ((token>>ML_RUN_BITS2) == 2) + { + offset = MEM_readLE24(ip); ip+=3; + } + else // (token>>ML_RUN_BITS2) == 3 + { + offset = last_off; + } +#endif + + last_off = offset; + match = op - offset; + if ((checkOffset) && (unlikely(match < lowLimit))) goto _output_error; /* Error : offset outside buffers */ + + /* get matchlength */ + length = token & ML_MASK; + if (length == ML_MASK) + { + unsigned s; + do + { + if ((endOnInput) && (ip > iend-LASTLITERALS)) goto _output_error; + s = *ip++; + length += s; + } while (s==255); + if ((safeDecode) && unlikely((size_t)(op+length)<(size_t)op)) goto _output_error; /* overflow detection */ + } + length += MINMATCH; + + /* check external dictionary */ + if ((dict==usingExtDict) && (match < lowPrefix)) + { + if (unlikely(op+length > oend-LASTLITERALS)) goto _output_error; /* doesn't respect parsing restriction */ + + if (length <= (size_t)(lowPrefix-match)) + { + /* match can be copied as a single segment from external dictionary */ + match = dictEnd - (lowPrefix-match); + memmove(op, match, length); op += length; + } + else + { + /* match encompass external dictionary and current block */ + size_t copySize = (size_t)(lowPrefix-match); + memcpy(op, dictEnd - copySize, copySize); + op += copySize; + copySize = length - copySize; + if (copySize > (size_t)(op-lowPrefix)) /* overlap copy */ + { + BYTE* const endOfMatch = op + copySize; + const BYTE* copyFrom = lowPrefix; + while (op < endOfMatch) *op++ = *copyFrom++; + } + else + { + memcpy(op, lowPrefix, copySize); + op += copySize; + } + } + continue; + } + + /* copy match within block */ + cpy = op + length; + if (unlikely(offset<8)) + { + const int dec64 = dec64table[offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[offset]; + memcpy(op+4, match, 4); + match -= dec64; + } else { MEM_copy8(op, match); match+=8; } + op += 8; + + if (unlikely(cpy>oend-(16-MINMATCH))) + { + BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1); + if (cpy > oend-LASTLITERALS) goto _output_error; /* Error : last LASTLITERALS bytes must be literals (uncompressed) */ + if (op < oCopyLimit) + { + MEM_wildCopy(op, match, oCopyLimit); + match += oCopyLimit - op; + op = oCopyLimit; + } + while (opprefixSize = (size_t) dictSize; + lz5sd->prefixEnd = (const BYTE*) dictionary + dictSize; + lz5sd->externalDict = NULL; + lz5sd->extDictSize = 0; + return 1; +} + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks must still be available at the memory position where they were decoded. + If it's not possible, save the relevant part of decoded data into a safe buffer, + and indicate where it stands using LZ5_setStreamDecode() +*/ +int LZ5_decompress_safe_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + LZ5_streamDecode_t_internal* lz5sd = (LZ5_streamDecode_t_internal*) LZ5_streamDecode; + int result; + + if (lz5sd->prefixEnd == (BYTE*)dest) + { + result = LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, lz5sd->prefixEnd - lz5sd->prefixSize, lz5sd->externalDict, lz5sd->extDictSize); + if (result <= 0) return result; + lz5sd->prefixSize += result; + lz5sd->prefixEnd += result; + } + else + { + lz5sd->extDictSize = lz5sd->prefixSize; + lz5sd->externalDict = lz5sd->prefixEnd - lz5sd->extDictSize; + result = LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, + endOnInputSize, full, 0, + usingExtDict, (BYTE*)dest, lz5sd->externalDict, lz5sd->extDictSize); + if (result <= 0) return result; + lz5sd->prefixSize = result; + lz5sd->prefixEnd = (BYTE*)dest + result; + } + + return result; +} + +int LZ5_decompress_fast_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int originalSize) +{ + LZ5_streamDecode_t_internal* lz5sd = (LZ5_streamDecode_t_internal*) LZ5_streamDecode; + int result; + + if (lz5sd->prefixEnd == (BYTE*)dest) + { + result = LZ5_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, lz5sd->prefixEnd - lz5sd->prefixSize, lz5sd->externalDict, lz5sd->extDictSize); + if (result <= 0) return result; + lz5sd->prefixSize += originalSize; + lz5sd->prefixEnd += originalSize; + } + else + { + lz5sd->extDictSize = lz5sd->prefixSize; + lz5sd->externalDict = (BYTE*)dest - lz5sd->extDictSize; + result = LZ5_decompress_generic(source, dest, 0, originalSize, + endOnOutputSize, full, 0, + usingExtDict, (BYTE*)dest, lz5sd->externalDict, lz5sd->extDictSize); + if (result <= 0) return result; + lz5sd->prefixSize = originalSize; + lz5sd->prefixEnd = (BYTE*)dest + originalSize; + } + + return result; +} + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as "_continue" ones, + the dictionary must be explicitly provided within parameters +*/ + +FORCE_INLINE int LZ5_decompress_usingDict_generic(const char* source, char* dest, int compressedSize, int maxOutputSize, int safe, const char* dictStart, int dictSize) +{ + if (dictSize==0) + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest, NULL, 0); + if (dictStart+dictSize == dest) + { + if (dictSize >= (int)(LZ5_DICT_SIZE - 1)) + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, withPrefix64k, (BYTE*)dest-LZ5_DICT_SIZE, NULL, 0); + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, noDict, (BYTE*)dest-dictSize, NULL, 0); + } + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, safe, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + +int LZ5_decompress_safe_usingDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ5_decompress_usingDict_generic(source, dest, compressedSize, maxOutputSize, 1, dictStart, dictSize); +} + +int LZ5_decompress_fast_usingDict(const char* source, char* dest, int originalSize, const char* dictStart, int dictSize) +{ + return LZ5_decompress_usingDict_generic(source, dest, 0, originalSize, 0, dictStart, dictSize); +} + +/* debug function */ +int LZ5_decompress_safe_forceExtDict(const char* source, char* dest, int compressedSize, int maxOutputSize, const char* dictStart, int dictSize) +{ + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, usingExtDict, (BYTE*)dest, (const BYTE*)dictStart, dictSize); +} + + +/*************************************************** +* Obsolete Functions +***************************************************/ +/* obsolete compression functions */ +int LZ5_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) { return LZ5_compress_default(source, dest, inputSize, maxOutputSize); } +int LZ5_compress(const char* source, char* dest, int inputSize) { return LZ5_compress_default(source, dest, inputSize, LZ5_compressBound(inputSize)); } +int LZ5_compress_limitedOutput_withState (void* state, const char* src, char* dst, int srcSize, int dstSize) { return LZ5_compress_fast_extState(state, src, dst, srcSize, dstSize, 1); } +int LZ5_compress_withState (void* state, const char* src, char* dst, int srcSize) { return LZ5_compress_fast_extState(state, src, dst, srcSize, LZ5_compressBound(srcSize), 1); } +int LZ5_compress_limitedOutput_continue (LZ5_stream_t* LZ5_stream, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_fast_continue(LZ5_stream, src, dst, srcSize, maxDstSize, 1); } +int LZ5_compress_continue (LZ5_stream_t* LZ5_stream, const char* source, char* dest, int inputSize) { return LZ5_compress_fast_continue(LZ5_stream, source, dest, inputSize, LZ5_compressBound(inputSize), 1); } + +/* +These function names are deprecated and should no longer be used. +They are only provided here for compatibility with older user programs. +- LZ5_uncompress is totally equivalent to LZ5_decompress_fast +- LZ5_uncompress_unknownOutputSize is totally equivalent to LZ5_decompress_safe +*/ +int LZ5_uncompress (const char* source, char* dest, int outputSize) { return LZ5_decompress_fast(source, dest, outputSize); } +int LZ5_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize) { return LZ5_decompress_safe(source, dest, isize, maxOutputSize); } + + +/* Obsolete Streaming functions */ + +int LZ5_sizeofStreamState() { return LZ5_STREAMSIZE; } + +static void LZ5_init(LZ5_stream_t_internal* lz5ds, BYTE* base) +{ + MEM_INIT(lz5ds, 0, LZ5_STREAMSIZE); + lz5ds->bufferStart = base; +} + +int LZ5_resetStreamState(void* state, char* inputBuffer) +{ + if ((((size_t)state) & 3) != 0) return 1; /* Error : pointer is not aligned on 4-bytes boundary */ + LZ5_init((LZ5_stream_t_internal*)state, (BYTE*)inputBuffer); + return 0; +} + +void* LZ5_create (char* inputBuffer) +{ + void* lz5ds = ALLOCATOR(8, LZ5_STREAMSIZE_U64); + LZ5_init ((LZ5_stream_t_internal*)lz5ds, (BYTE*)inputBuffer); + return lz5ds; +} + +char* LZ5_slideInputBuffer (void* LZ5_Data) +{ + LZ5_stream_t_internal* ctx = (LZ5_stream_t_internal*)LZ5_Data; + int dictSize = LZ5_saveDict((LZ5_stream_t*)LZ5_Data, (char*)ctx->bufferStart, LZ5_DICT_SIZE); + return (char*)(ctx->bufferStart + dictSize); +} + +/* Obsolete streaming decompression functions */ + +int LZ5_decompress_safe_withPrefix64k(const char* source, char* dest, int compressedSize, int maxOutputSize) +{ + return LZ5_decompress_generic(source, dest, compressedSize, maxOutputSize, endOnInputSize, full, 0, withPrefix64k, (BYTE*)dest - LZ5_DICT_SIZE, NULL, LZ5_DICT_SIZE); +} + +int LZ5_decompress_fast_withPrefix64k(const char* source, char* dest, int originalSize) +{ + return LZ5_decompress_generic(source, dest, 0, originalSize, endOnOutputSize, full, 0, withPrefix64k, (BYTE*)dest - LZ5_DICT_SIZE, NULL, LZ5_DICT_SIZE); +} diff --git a/C/lz5/lz5.h b/C/lz5/lz5.h new file mode 100644 index 00000000..56243aa0 --- /dev/null +++ b/C/lz5/lz5.h @@ -0,0 +1,363 @@ +/* + LZ5 - Fast LZ compression algorithm + Header File + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ5 source repository : https://github.com/inikep/lz5 + - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* + * lz5.h provides block compression functions, and gives full buffer control to programmer. + * If you need to generate inter-operable compressed data (respecting LZ5 frame specification), + * and can let the library handle its own memory, please use lz5frame.h instead. +*/ + +/************************************** +* Version +**************************************/ +#define LZ5_VERSION "v1.5.0" +#define LZ5_VERSION_MAJOR 1 /* for breaking interface changes */ +#define LZ5_VERSION_MINOR 5 /* for new (non-breaking) interface capabilities */ +#define LZ5_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */ +#define LZ5_VERSION_NUMBER (LZ5_VERSION_MAJOR *100*100 + LZ5_VERSION_MINOR *100 + LZ5_VERSION_RELEASE) +int LZ5_versionNumber (void); + +#define LZ5HC_MAX_CLEVEL 15 + + +/************************************** +* Tuning parameter +**************************************/ +/* + * LZ5_MEMORY_USAGE : + * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) + * Increasing memory usage improves compression ratio + * Reduced memory usage can improve speed, due to cache effect + */ +#define LZ5_MEMORY_USAGE 20 + + +/************************************** +* Simple Functions +**************************************/ + +int LZ5_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize); +int LZ5_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize); + +/* +LZ5_compress_default() : + Compresses 'sourceSize' bytes from buffer 'source' + into already allocated 'dest' buffer of size 'maxDestSize'. + Compression is guaranteed to succeed if 'maxDestSize' >= LZ5_compressBound(sourceSize). + It also runs faster, so it's a recommended setting. + If the function cannot compress 'source' into a more limited 'dest' budget, + compression stops *immediately*, and the function result is zero. + As a consequence, 'dest' content is not valid. + This function never writes outside 'dest' buffer, nor read outside 'source' buffer. + sourceSize : Max supported value is LZ5_MAX_INPUT_VALUE + maxDestSize : full or partial size of buffer 'dest' (which must be already allocated) + return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize) + or 0 if compression fails + +LZ5_decompress_safe() : + compressedSize : is the precise full size of the compressed block. + maxDecompressedSize : is the size of destination buffer, which must be already allocated. + return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize) + If destination buffer is not large enough, decoding will stop and output an error code (<0). + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function is protected against buffer overflow exploits, including malicious data packets. + It never writes outside output buffer, nor reads outside input buffer. +*/ + + +/************************************** +* Advanced Functions +**************************************/ +#define LZ5_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */ +#define LZ5_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ5_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/128) + 16) + +/* +LZ5_compressBound() : + Provides the maximum size that LZ5 compression may output in a "worst case" scenario (input data not compressible) + This function is primarily useful for memory allocation purposes (destination buffer size). + Macro LZ5_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example). + Note that LZ5_compress_default() compress faster when dest buffer size is >= LZ5_compressBound(srcSize) + inputSize : max supported value is LZ5_MAX_INPUT_SIZE + return : maximum output size in a "worst case" scenario + or 0, if input size is too large ( > LZ5_MAX_INPUT_SIZE) +*/ +int LZ5_compressBound(int inputSize); + +/* +LZ5_compress_fast() : + Same as LZ5_compress_default(), but allows to select an "acceleration" factor. + The larger the acceleration value, the faster the algorithm, but also the lesser the compression. + It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed. + An acceleration value of "1" is the same as regular LZ5_compress_default() + Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz5.c), which is 1. +*/ +int LZ5_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration); + + +/* +LZ5_compress_fast_extState() : + Same compression function, just using an externally allocated memory space to store compression state. + Use LZ5_sizeofState() to know how much memory must be allocated, + and allocate it on 8-bytes boundaries (using malloc() typically). + Then, provide it as 'void* state' to compression function. +*/ +int LZ5_sizeofState(void); +int LZ5_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration); + + +/* +LZ5_compress_destSize() : + Reverse the logic, by compressing as much data as possible from 'source' buffer + into already allocated buffer 'dest' of size 'targetDestSize'. + This function either compresses the entire 'source' content into 'dest' if it's large enough, + or fill 'dest' buffer completely with as much data as possible from 'source'. + *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'. + New value is necessarily <= old value. + return : Nb bytes written into 'dest' (necessarily <= targetDestSize) + or 0 if compression fails +*/ +int LZ5_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize); + + +/* +LZ5_decompress_fast() : + originalSize : is the original and therefore uncompressed size + return : the number of bytes read from the source buffer (in other words, the compressed size) + If the source stream is detected malformed, the function will stop decoding and return a negative result. + Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes. + note : This function fully respect memory boundaries for properly formed compressed data. + It is a bit faster than LZ5_decompress_safe(). + However, it does not provide any protection against intentionally modified data stream (malicious input). + Use this function in trusted environment only (data to decode comes from a trusted source). +*/ +int LZ5_decompress_fast (const char* source, char* dest, int originalSize); + +/* +LZ5_decompress_safe_partial() : + This function decompress a compressed block of size 'compressedSize' at position 'source' + into destination buffer 'dest' of size 'maxDecompressedSize'. + The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached, + reducing decompression time. + return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize) + Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller. + Always control how many bytes were decoded. + If the source stream is detected malformed, the function will stop decoding and return a negative result. + This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets +*/ +int LZ5_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize); + + +/*********************************************** +* Streaming Compression Functions +***********************************************/ +#define LZ5_STREAMSIZE_U64 ((1 << (LZ5_MEMORY_USAGE-3)) + 4) +#define LZ5_STREAMSIZE (LZ5_STREAMSIZE_U64 * sizeof(long long)) +/* + * LZ5_stream_t + * information structure to track an LZ5 stream. + * important : init this structure content before first use ! + * note : only allocated directly the structure if you are statically linking LZ5 + * If you are using liblz5 as a DLL, please use below construction methods instead. + */ +typedef struct { long long table[LZ5_STREAMSIZE_U64]; } LZ5_stream_t; + +/* + * LZ5_resetStream + * Use this function to init an allocated LZ5_stream_t structure + */ +void LZ5_resetStream (LZ5_stream_t* streamPtr); + +/* + * LZ5_createStream will allocate and initialize an LZ5_stream_t structure + * LZ5_freeStream releases its memory. + * In the context of a DLL (liblz5), please use these methods rather than the static struct. + * They are more future proof, in case of a change of LZ5_stream_t size. + */ +LZ5_stream_t* LZ5_createStream(void); +int LZ5_freeStream (LZ5_stream_t* streamPtr); + +/* + * LZ5_loadDict + * Use this function to load a static dictionary into LZ5_stream. + * Any previous data will be forgotten, only 'dictionary' will remain in memory. + * Loading a size of 0 is allowed. + * Return : dictionary size, in bytes (necessarily <= 64 KB) + */ +int LZ5_loadDict (LZ5_stream_t* streamPtr, const char* dictionary, int dictSize); + +/* + * LZ5_compress_fast_continue + * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio. + * Important : Previous data blocks are assumed to still be present and unmodified ! + * 'dst' buffer must be already allocated. + * If maxDstSize >= LZ5_compressBound(srcSize), compression is guaranteed to succeed, and runs faster. + * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero. + */ +int LZ5_compress_fast_continue (LZ5_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration); + +/* + * LZ5_saveDict + * If previously compressed data block is not guaranteed to remain available at its memory location + * save it into a safer place (char* safeBuffer) + * Note : you don't need to call LZ5_loadDict() afterwards, + * dictionary is immediately usable, you can therefore call LZ5_compress_fast_continue() + * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error + */ +int LZ5_saveDict (LZ5_stream_t* streamPtr, char* safeBuffer, int dictSize); + + +/************************************************ +* Streaming Decompression Functions +************************************************/ + +#define LZ5_STREAMDECODESIZE_U64 4 +#define LZ5_STREAMDECODESIZE (LZ5_STREAMDECODESIZE_U64 * sizeof(unsigned long long)) +typedef struct { unsigned long long table[LZ5_STREAMDECODESIZE_U64]; } LZ5_streamDecode_t; +/* + * LZ5_streamDecode_t + * information structure to track an LZ5 stream. + * init this structure content using LZ5_setStreamDecode or memset() before first use ! + * + * In the context of a DLL (liblz5) please prefer usage of construction methods below. + * They are more future proof, in case of a change of LZ5_streamDecode_t size in the future. + * LZ5_createStreamDecode will allocate and initialize an LZ5_streamDecode_t structure + * LZ5_freeStreamDecode releases its memory. + */ +LZ5_streamDecode_t* LZ5_createStreamDecode(void); +int LZ5_freeStreamDecode (LZ5_streamDecode_t* LZ5_stream); + +/* + * LZ5_setStreamDecode + * Use this function to instruct where to find the dictionary. + * Setting a size of 0 is allowed (same effect as reset). + * Return : 1 if OK, 0 if error + */ +int LZ5_setStreamDecode (LZ5_streamDecode_t* LZ5_streamDecode, const char* dictionary, int dictSize); + +/* +*_continue() : + These decoding functions allow decompression of multiple blocks in "streaming" mode. + Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB) + In the case of a ring buffers, decoding buffer must be either : + - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions) + In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB). + - Larger than encoding buffer, by a minimum of maxBlockSize more bytes. + maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including small ones ( < 64 KB). + - _At least_ 64 KB + 8 bytes + maxBlockSize. + In which case, encoding and decoding buffers do not need to be synchronized, + and encoding ring buffer can have any size, including larger than decoding buffer. + Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer, + and indicate where it is saved using LZ5_setStreamDecode() +*/ +int LZ5_decompress_safe_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize); +int LZ5_decompress_fast_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int originalSize); + + +/* +Advanced decoding functions : +*_usingDict() : + These decoding functions work the same as + a combination of LZ5_setStreamDecode() followed by LZ5_decompress_x_continue() + They are stand-alone. They don't need nor update an LZ5_streamDecode_t structure. +*/ +int LZ5_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize); +int LZ5_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize); + + +/************************************** +* Obsolete Functions +**************************************/ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + Otherwise, you can also define LZ5_DISABLE_DEPRECATE_WARNINGS */ +#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#ifdef LZ5_DISABLE_DEPRECATE_WARNINGS +# define LZ5_DEPRECATED() /* disable deprecation warnings */ +#else +# if (LZ5_GCC_VERSION >= 405) || defined(__clang__) +# define LZ5_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ5_GCC_VERSION >= 301) +# define LZ5_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ5_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ5_DEPRECATED for this compiler") +# define LZ5_DEPRECATED(message) +# endif +#endif /* LZ5_DISABLE_DEPRECATE_WARNINGS */ + +/* Obsolete compression functions */ +/* These functions will generate warnings in a future release */ +int LZ5_compress (const char* source, char* dest, int sourceSize); +int LZ5_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize); +int LZ5_compress_withState (void* state, const char* source, char* dest, int inputSize); +int LZ5_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ5_compress_continue (LZ5_stream_t* LZ5_streamPtr, const char* source, char* dest, int inputSize); +int LZ5_compress_limitedOutput_continue (LZ5_stream_t* LZ5_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize); + +/* Obsolete decompression functions */ +/* These function names are completely deprecated and must no longer be used. + They are only provided in lz5.c for compatibility with older programs. + - LZ5_uncompress is the same as LZ5_decompress_fast + - LZ5_uncompress_unknownOutputSize is the same as LZ5_decompress_safe + These function prototypes are now disabled; uncomment them only if you really need them. + It is highly recommended to stop using these prototypes and migrate to maintained ones */ +/* int LZ5_uncompress (const char* source, char* dest, int outputSize); */ +/* int LZ5_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */ + +/* Obsolete streaming functions; use new streaming interface whenever possible */ +LZ5_DEPRECATED("use LZ5_createStream() instead") void* LZ5_create (char* inputBuffer); +LZ5_DEPRECATED("use LZ5_createStream() instead") int LZ5_sizeofStreamState(void); +LZ5_DEPRECATED("use LZ5_resetStream() instead") int LZ5_resetStreamState(void* state, char* inputBuffer); +LZ5_DEPRECATED("use LZ5_saveDict() instead") char* LZ5_slideInputBuffer (void* state); + +/* Obsolete streaming decoding functions */ +LZ5_DEPRECATED("use LZ5_decompress_safe_usingDict() instead") int LZ5_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize); +LZ5_DEPRECATED("use LZ5_decompress_fast_usingDict() instead") int LZ5_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize); + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz5/lz5common.h b/C/lz5/lz5common.h new file mode 100644 index 00000000..41e42b71 --- /dev/null +++ b/C/lz5/lz5common.h @@ -0,0 +1,330 @@ +#ifndef LZ5COMMON_H +#define LZ5COMMON_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +/************************************** +* Tuning parameters +**************************************/ +/* + * HEAPMODE : + * Select how default compression functions will allocate memory for their hash table, + * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()). + */ +#ifdef _MSC_VER + #define HEAPMODE 1 /* Default stack size for VC++ is 1 MB and size of LZ5_stream_t exceeds that limit */ +#else + #define HEAPMODE 0 +#endif + + +/* + * ACCELERATION_DEFAULT : + * Select "acceleration" for LZ5_compress_fast() when parameter value <= 0 + */ +#define ACCELERATION_DEFAULT 1 + + + + +/************************************** +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */ +#else +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# if defined(__GNUC__) || defined(__clang__) +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif /* _MSC_VER */ + +#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if (LZ5_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + + +/************************************** +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(n,s) calloc(n,s) +#define FREEMEM free +#include /* memset, memcpy */ +#define MEM_INIT memset + + +/************************************** +* Common Constants +**************************************/ +#define MINMATCH 3 // should be 3 or 4 + +#define WILDCOPYLENGTH 8 +#define LASTLITERALS 5 +#define MFLIMIT (WILDCOPYLENGTH+MINMATCH) +static const int LZ5_minLength = (MFLIMIT+1); + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define MAXD_LOG 22 +#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) +#define LZ5_DICT_SIZE (1 << MAXD_LOG) + +#define ML_BITS 3 +#define ML_MASK ((1U<= 199901L) /* C99 */) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/* ************************************* +* HC Inline functions and Macros +***************************************/ +#include "mem.h" // MEM_read +#include "lz5.h" // LZ5HC_MAX_CLEVEL + + +static const U32 prime4bytes = 2654435761U; +static const U64 prime5bytes = 889523592379ULL; + +#ifdef LZ5HC_INCLUDES +static const U32 prime3bytes = 506832829U; +static const U64 prime6bytes = 227718039650203ULL; +static const U64 prime7bytes = 58295818150454627ULL; + +static U32 LZ5HC_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; } +static size_t LZ5HC_hash3Ptr(const void* ptr, U32 h) { return LZ5HC_hash3(MEM_read32(ptr), h); } + +static U32 LZ5HC_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t LZ5HC_hash4Ptr(const void* ptr, U32 h) { return LZ5HC_hash4(MEM_read32(ptr), h); } + +static size_t LZ5HC_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; } +static size_t LZ5HC_hash5Ptr(const void* p, U32 h) { return LZ5HC_hash5(MEM_read64(p), h); } + +static size_t LZ5HC_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; } +static size_t LZ5HC_hash6Ptr(const void* p, U32 h) { return LZ5HC_hash6(MEM_read64(p), h); } + +static size_t LZ5HC_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; } +static size_t LZ5HC_hash7Ptr(const void* p, U32 h) { return LZ5HC_hash7(MEM_read64(p), h); } + +static size_t LZ5HC_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return LZ5HC_hash4Ptr(p, hBits); + case 5: return LZ5HC_hash5Ptr(p, hBits); + case 6: return LZ5HC_hash6Ptr(p, hBits); + case 7: return LZ5HC_hash7Ptr(p, hBits); + } +} + + +/************************************** +* HC Local Macros +**************************************/ +#define LZ5HC_DEBUG(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ5_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ5_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__) +#define LZ5_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__) + +#define MAX(a,b) ((a)>(b))?(a):(b) +#define LZ5_OPT_NUM (1<<12) + +#define LZ5_SHORT_LITERALS ((1< LZ5_MID_OFFSET_DISTANCE) || (offset 0) + sum = MAX(common + literals, best_common); + else + sum = MAX(common, best_common - literals); + +// return LZ5_CODEWORD_COST(sum - common, (off == last_off) ? 0 : (off), common - MINMATCH) <= LZ5_CODEWORD_COST(sum - best_common, (best_off == last_off) ? 0 : (best_off), best_common - MINMATCH); + return LZ5_NORMAL_MATCH_COST(common - MINMATCH, (off == last_off) ? 0 : off) + LZ5_NORMAL_LIT_COST(sum - common) <= LZ5_NORMAL_MATCH_COST(best_common - MINMATCH, (best_off == last_off) ? 0 : (best_off)) + LZ5_NORMAL_LIT_COST(sum - best_common); +} + +#endif // LZ5HC_INCLUDES + + + +/* ************************************* +* HC Types +***************************************/ +/** from faster to stronger */ +typedef enum { LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price, LZ5HC_optimal_price, LZ5HC_optimal_price_bt } LZ5HC_strategy; + +typedef struct +{ + U32 windowLog; /* largest match distance : impact decompression buffer size */ + U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */ + U32 hashLog; /* dispatch table : larger == more memory, faster*/ + U32 hashLog3; /* dispatch table : larger == more memory, faster*/ + U32 searchNum; /* nb of searches : larger == more compression, slower*/ + U32 searchLength; /* size of matches : larger == faster decompression */ + U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */ + U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */ + LZ5HC_strategy strategy; +} LZ5HC_parameters; + + +struct LZ5HC_Data_s +{ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + const BYTE* end; /* next block here to continue on current prefix */ + const BYTE* base; /* All index relative to this position */ + const BYTE* dictBase; /* alternate base for extDict */ + const BYTE* inputBuffer; /* for debugging */ + const BYTE* outputBuffer; /* for debugging */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more dict */ + U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 compressionLevel; + U32 last_off; + LZ5HC_parameters params; +}; + +typedef struct +{ + int off; + int len; + int back; +} LZ5HC_match_t; + +typedef struct +{ + int price; + int off; + int mlen; + int litlen; + int rep; +} LZ5HC_optimal_t; + + + +/* ************************************* +* HC Pre-defined compression levels +***************************************/ + +static const int g_maxCompressionLevel = LZ5HC_MAX_CLEVEL; +static const int LZ5HC_compressionLevel_default = 6; + +static const LZ5HC_parameters LZ5HC_defaultParameters[LZ5HC_MAX_CLEVEL+1] = +{ + /* windLog, contentLog, H, H3, Snum, SL, SuffL, FS, Strategy */ + { 0, 0, 0, 0, 0, 0, 0, 0, LZ5HC_fast }, // level 0 - never used + { MAXD_LOG, MAXD_LOG, 13, 0, 4, 6, 0, 0, LZ5HC_fast }, // level 1 + { MAXD_LOG, MAXD_LOG, 13, 0, 2, 6, 0, 0, LZ5HC_fast }, // level 2 + { MAXD_LOG, MAXD_LOG, 13, 0, 1, 5, 0, 0, LZ5HC_fast }, // level 3 + { MAXD_LOG, MAXD_LOG, 14, 13, 1, 4, 0, 0, LZ5HC_price_fast }, // level 4 + { MAXD_LOG, MAXD_LOG, 17, 13, 1, 4, 0, 0, LZ5HC_price_fast }, // level 5 + { MAXD_LOG, MAXD_LOG, 15, 13, 1, 4, 0, 0, LZ5HC_lowest_price }, // level 6 + { MAXD_LOG, MAXD_LOG, 17, 13, 1, 4, 0, 0, LZ5HC_lowest_price }, // level 7 + { MAXD_LOG, MAXD_LOG, 19, 16, 1, 4, 0, 0, LZ5HC_lowest_price }, // level 8 + { MAXD_LOG, MAXD_LOG, 23, 16, 3, 4, 0, 0, LZ5HC_lowest_price }, // level 9 + { MAXD_LOG, MAXD_LOG, 23, 16, 8, 4, 0, 0, LZ5HC_lowest_price }, // level 10 + { MAXD_LOG, MAXD_LOG, 23, 16, 8, 4, 12, 0, LZ5HC_optimal_price }, // level 11 + { MAXD_LOG, MAXD_LOG, 23, 16, 8, 4, 64, 0, LZ5HC_optimal_price }, // level 12 + { MAXD_LOG, MAXD_LOG+1, 23, 16, 8, 4, 64, 1, LZ5HC_optimal_price_bt }, // level 13 + { MAXD_LOG, MAXD_LOG+1, 23, 16, 128, 4, 64, 1, LZ5HC_optimal_price_bt }, // level 14 + { MAXD_LOG, MAXD_LOG+1, 28, 24, 1<<10, 4, 1<<10, 1, LZ5HC_optimal_price_bt }, // level 15 +// { 10, 10, 10, 0, 0, 4, 0, 0, LZ5HC_fast }, // min values +// { 24, 24, 28, 24, 1<<24, 7, 1<<24, 2, LZ5HC_optimal_price }, // max values +}; + + + +#if defined (__cplusplus) +} +#endif + +#endif /* LZ5COMMON_H */ diff --git a/C/lz5/lz5frame.c b/C/lz5/lz5frame.c new file mode 100644 index 00000000..f6e4773b --- /dev/null +++ b/C/lz5/lz5frame.c @@ -0,0 +1,1485 @@ +/* +LZ5 auto-framing library +Copyright (C) 2011-2015, Yann Collet. + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ5 source repository : https://github.com/inikep/lz5 +- LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ + +/* LZ5F is a stand-alone API to create LZ5-compressed Frames +* in full conformance with specification v1.5.0 +* All related operations, including memory management, are handled by the library. +* */ + + +/************************************** +* Compiler Options +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/************************************** +* Memory routines +**************************************/ +#include /* malloc, calloc, free */ +#define ALLOCATOR(s) calloc(1,s) +#define FREEMEM free +#include /* memset, memcpy, memmove */ +#include +#define MEM_INIT memset + + +/************************************** +* Includes +**************************************/ +#include "lz5frame_static.h" +#include "lz5.h" +#include "lz5hc.h" +#include "xxhash.h" + + +/************************************** +* Basic Types +**************************************/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */ +# include +typedef uint8_t BYTE; +typedef uint16_t U16; +typedef uint32_t U32; +typedef int32_t S32; +typedef uint64_t U64; +#else +typedef unsigned char BYTE; +typedef unsigned short U16; +typedef unsigned int U32; +typedef signed int S32; +typedef unsigned long long U64; +#endif + + +/************************************** +* Constants +**************************************/ +#define KB *(1<<10) +#define MB *(1<<20) +#define GB *(1<<30) + +#define _1BIT 0x01 +#define _2BITS 0x03 +#define _3BITS 0x07 +#define _4BITS 0x0F +#define _8BITS 0xFF + +#define LZ5F_MAGIC_SKIPPABLE_START 0x184D2A50U +#define LZ5F_MAGICNUMBER 0x184D2205U +#define LZ5F_BLOCKUNCOMPRESSED_FLAG 0x80000000U +#define LZ5F_BLOCKSIZEID_DEFAULT LZ5F_max64KB +#define LZ5F_DICT_SIZE (1 << 22) + +static const size_t minFHSize = 7; +static const size_t maxFHSize = 15; +static const size_t BHSize = 4; +static const int minHClevel = 1; + + +/************************************** +* Structures and local types +**************************************/ +typedef struct LZ5F_cctx_s +{ + LZ5F_preferences_t prefs; + U32 version; + U32 cStage; + size_t maxBlockSize; + size_t maxBufferSize; + BYTE* tmpBuff; + BYTE* tmpIn; + size_t tmpInSize; + U64 totalInSize; + XXH32_state_t xxh; + void* lz5CtxPtr; + U32 lz5CtxLevel; /* 0: unallocated; 1: LZ5_stream_t; 3: LZ5_streamHC_t */ +} LZ5F_cctx_t; + +typedef struct LZ5F_dctx_s +{ + LZ5F_frameInfo_t frameInfo; + U32 version; + U32 dStage; + U64 frameRemainingSize; + size_t maxBlockSize; + size_t maxBufferSize; + const BYTE* srcExpect; + BYTE* tmpIn; + size_t tmpInSize; + size_t tmpInTarget; + BYTE* tmpOutBuffer; + const BYTE* dict; + size_t dictSize; + BYTE* tmpOut; + size_t tmpOutSize; + size_t tmpOutStart; + XXH32_state_t xxh; + BYTE header[16]; +} LZ5F_dctx_t; + + +/************************************** +* Error management +**************************************/ +#define LZ5F_GENERATE_STRING(STRING) #STRING, +static const char* LZ5F_errorStrings[] = { LZ5F_LIST_ERRORS(LZ5F_GENERATE_STRING) }; + + +unsigned LZ5F_isError(LZ5F_errorCode_t code) +{ + return (code > (LZ5F_errorCode_t)(-LZ5F_ERROR_maxCode)); +} + +const char* LZ5F_getErrorName(LZ5F_errorCode_t code) +{ + static const char* codeError = "Unspecified error code"; + if (LZ5F_isError(code)) return LZ5F_errorStrings[-(int)(code)]; + return codeError; +} + + +/************************************** +* Private functions +**************************************/ +static size_t LZ5F_getBlockSize(unsigned blockSizeID) +{ + static const size_t blockSizes[7] = { 64 KB, 256 KB, 1 MB, 4 MB, 16 MB, 64 MB, 256 MB }; + + if (blockSizeID == 0) blockSizeID = LZ5F_BLOCKSIZEID_DEFAULT; + blockSizeID -= 1; + if (blockSizeID >= 7) return (size_t)-LZ5F_ERROR_maxBlockSize_invalid; + + // printf("LZ5F_getBlockSize %d %d\n", blockSizeID+1, (int)blockSizes[blockSizeID]); + return blockSizes[blockSizeID]; +} + + +/* unoptimized version; solves endianess & alignment issues */ +static U32 LZ5F_readLE32 (const BYTE* srcPtr) +{ + U32 value32 = srcPtr[0]; + value32 += (srcPtr[1]<<8); + value32 += (srcPtr[2]<<16); + value32 += ((U32)srcPtr[3])<<24; + return value32; +} + +static void LZ5F_writeLE32 (BYTE* dstPtr, U32 value32) +{ + dstPtr[0] = (BYTE)value32; + dstPtr[1] = (BYTE)(value32 >> 8); + dstPtr[2] = (BYTE)(value32 >> 16); + dstPtr[3] = (BYTE)(value32 >> 24); +} + +static U64 LZ5F_readLE64 (const BYTE* srcPtr) +{ + U64 value64 = srcPtr[0]; + value64 += ((U64)srcPtr[1]<<8); + value64 += ((U64)srcPtr[2]<<16); + value64 += ((U64)srcPtr[3]<<24); + value64 += ((U64)srcPtr[4]<<32); + value64 += ((U64)srcPtr[5]<<40); + value64 += ((U64)srcPtr[6]<<48); + value64 += ((U64)srcPtr[7]<<56); + return value64; +} + +static void LZ5F_writeLE64 (BYTE* dstPtr, U64 value64) +{ + dstPtr[0] = (BYTE)value64; + dstPtr[1] = (BYTE)(value64 >> 8); + dstPtr[2] = (BYTE)(value64 >> 16); + dstPtr[3] = (BYTE)(value64 >> 24); + dstPtr[4] = (BYTE)(value64 >> 32); + dstPtr[5] = (BYTE)(value64 >> 40); + dstPtr[6] = (BYTE)(value64 >> 48); + dstPtr[7] = (BYTE)(value64 >> 56); +} + + +static BYTE LZ5F_headerChecksum (const void* header, size_t length) +{ + U32 xxh = XXH32(header, length, 0); + return (BYTE)(xxh >> 8); +} + + +/************************************** +* Simple compression functions +**************************************/ +static LZ5F_blockSizeID_t LZ5F_optimalBSID(const LZ5F_blockSizeID_t requestedBSID, const size_t srcSize) +{ + LZ5F_blockSizeID_t proposedBSID = LZ5F_max64KB; + size_t maxBlockSize = 64 KB; + while (requestedBSID > proposedBSID) + { + if (srcSize <= maxBlockSize) + return proposedBSID; + proposedBSID = (LZ5F_blockSizeID_t)((int)proposedBSID + 1); + maxBlockSize <<= 2; + } + return requestedBSID; +} + + +void LZ5F_freeStream(LZ5F_cctx_t* cctxPtr) +{ + if (cctxPtr->lz5CtxLevel == 1) + LZ5_freeStream((LZ5_stream_t*)cctxPtr->lz5CtxPtr); + else if (cctxPtr->lz5CtxLevel == 2) + LZ5_freeStreamHC((LZ5_streamHC_t*)cctxPtr->lz5CtxPtr); + cctxPtr->lz5CtxLevel = 0; +} + + +size_t LZ5F_compressFrameBound(size_t srcSize, const LZ5F_preferences_t* preferencesPtr) +{ + LZ5F_preferences_t prefs; + size_t headerSize; + size_t streamSize; + + if (preferencesPtr!=NULL) prefs = *preferencesPtr; + else memset(&prefs, 0, sizeof(prefs)); + + prefs.frameInfo.blockSizeID = LZ5F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + + headerSize = maxFHSize; /* header size, including magic number and frame content size*/ + streamSize = LZ5F_compressBound(srcSize, &prefs); + + return headerSize + streamSize; +} + + +/* LZ5F_compressFrame() +* Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.0, in a single step. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* You can get the minimum value of dstMaxSize by using LZ5F_compressFrameBound() +* If this condition is not respected, LZ5F_compressFrame() will fail (result is an errorCode) +* The LZ5F_preferences_t structure is optional : you can provide NULL as argument. All preferences will then be set to default. +* The result of the function is the number of bytes written into dstBuffer. +* The function outputs an error code if it fails (can be tested using LZ5F_isError()) +*/ +size_t LZ5F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_preferences_t* preferencesPtr) +{ + LZ5F_cctx_t cctxI; + LZ5F_preferences_t prefs; + LZ5F_compressOptions_t options; + LZ5F_errorCode_t errorCode; + BYTE* const dstStart = (BYTE*) dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* const dstEnd = dstStart + dstMaxSize; + + memset(&cctxI, 0, sizeof(cctxI)); /* works because no allocation */ + memset(&options, 0, sizeof(options)); + + cctxI.version = LZ5F_VERSION; + cctxI.maxBufferSize = 5 MB; /* mess with real buffer size to prevent allocation; works because autoflush==1 & stableSrc==1 */ + + if (preferencesPtr!=NULL) + prefs = *preferencesPtr; + else + memset(&prefs, 0, sizeof(prefs)); + if (prefs.frameInfo.contentSize != 0) + prefs.frameInfo.contentSize = (U64)srcSize; /* auto-correct content size if selected (!=0) */ + + prefs.frameInfo.blockSizeID = LZ5F_optimalBSID(prefs.frameInfo.blockSizeID, srcSize); + prefs.autoFlush = 1; + if (srcSize <= LZ5F_getBlockSize(prefs.frameInfo.blockSizeID)) + prefs.frameInfo.blockMode = LZ5F_blockIndependent; /* no need for linked blocks */ + + options.stableSrc = 1; + + if (dstMaxSize < LZ5F_compressFrameBound(srcSize, &prefs)) + return (size_t)-LZ5F_ERROR_dstMaxSize_tooSmall; + + errorCode = LZ5F_compressBegin(&cctxI, dstBuffer, dstMaxSize, &prefs); /* write header */ + if (LZ5F_isError(errorCode)) return errorCode; + dstPtr += errorCode; /* header size */ + + errorCode = LZ5F_compressUpdate(&cctxI, dstPtr, dstEnd-dstPtr, srcBuffer, srcSize, &options); + if (LZ5F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + errorCode = LZ5F_compressEnd(&cctxI, dstPtr, dstEnd-dstPtr, &options); /* flush last block, and generate suffix */ + if (LZ5F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + LZ5F_freeStream(&cctxI); + + return (dstPtr - dstStart); +} + + +/*********************************** +* Advanced compression functions +***********************************/ + +/* LZ5F_createCompressionContext() : +* The first thing to do is to create a compressionContext object, which will be used in all compression operations. +* This is achieved using LZ5F_createCompressionContext(), which takes as argument a version and an LZ5F_preferences_t structure. +* The version provided MUST be LZ5F_VERSION. It is intended to track potential version differences between different binaries. +* The function will provide a pointer to an allocated LZ5F_compressionContext_t object. +* If the result LZ5F_errorCode_t is not OK_NoError, there was an error during context creation. +* Object can release its memory using LZ5F_freeCompressionContext(); +*/ +LZ5F_errorCode_t LZ5F_createCompressionContext(LZ5F_compressionContext_t* LZ5F_compressionContextPtr, unsigned version) +{ + LZ5F_cctx_t* cctxPtr; + + cctxPtr = (LZ5F_cctx_t*)ALLOCATOR(sizeof(LZ5F_cctx_t)); + if (cctxPtr==NULL) return (LZ5F_errorCode_t)(-LZ5F_ERROR_allocation_failed); + + cctxPtr->version = version; + cctxPtr->cStage = 0; /* Next stage : write header */ + + *LZ5F_compressionContextPtr = (LZ5F_compressionContext_t)cctxPtr; + + return LZ5F_OK_NoError; +} + + +LZ5F_errorCode_t LZ5F_freeCompressionContext(LZ5F_compressionContext_t LZ5F_compressionContext) +{ + LZ5F_cctx_t* cctxPtr = (LZ5F_cctx_t*)LZ5F_compressionContext; + + if (cctxPtr != NULL) /* null pointers can be safely provided to this function, like free() */ + { + LZ5F_freeStream(cctxPtr); + FREEMEM(cctxPtr->tmpBuff); + FREEMEM(LZ5F_compressionContext); + } + + return LZ5F_OK_NoError; +} + + +/* LZ5F_compressBegin() : +* will write the frame header into dstBuffer. +* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is LZ5F_MAXHEADERFRAME_SIZE bytes. +* The result of the function is the number of bytes written into dstBuffer for the header +* or an error code (can be tested using LZ5F_isError()) +*/ +size_t LZ5F_compressBegin(LZ5F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ5F_preferences_t* preferencesPtr) +{ + LZ5F_preferences_t prefNull; + LZ5F_cctx_t* cctxPtr = (LZ5F_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + BYTE* headerStart; + size_t requiredBuffSize; + + if (dstMaxSize < maxFHSize) return (size_t)-LZ5F_ERROR_dstMaxSize_tooSmall; + if (cctxPtr->cStage != 0) return (size_t)-LZ5F_ERROR_GENERIC; + memset(&prefNull, 0, sizeof(prefNull)); + if (preferencesPtr == NULL) preferencesPtr = &prefNull; + cctxPtr->prefs = *preferencesPtr; + cctxPtr->prefs.frameInfo.blockMode = LZ5F_blockIndependent; + + /* ctx Management */ + { + U32 tableID = (cctxPtr->prefs.compressionLevel < minHClevel) ? 1 : 2; /* 0:nothing ; 1:LZ5_createStream ; 2:LZ5_createStreamHC */ + // printf("BEFORE lz5CtxLevel=%d tableID=%d compressionLevel=%d minHClevel=%d\n", (int)cctxPtr->lz5CtxLevel, (int)tableID, (int)cctxPtr->prefs.compressionLevel, minHClevel); + if (cctxPtr->lz5CtxLevel != tableID) + { + LZ5F_freeStream(cctxPtr); + + cctxPtr->lz5CtxLevel = tableID; + if (cctxPtr->lz5CtxLevel == 1) + cctxPtr->lz5CtxPtr = (void*)LZ5_createStream(); + else + cctxPtr->lz5CtxPtr = (void*)LZ5_createStreamHC(cctxPtr->prefs.compressionLevel); + } + } + + /* Buffer Management */ + if (cctxPtr->prefs.frameInfo.blockSizeID == 0) cctxPtr->prefs.frameInfo.blockSizeID = LZ5F_BLOCKSIZEID_DEFAULT; + cctxPtr->maxBlockSize = LZ5F_getBlockSize(cctxPtr->prefs.frameInfo.blockSizeID); + + requiredBuffSize = cctxPtr->maxBlockSize + ((cctxPtr->prefs.frameInfo.blockMode == LZ5F_blockLinked) * 2 * LZ5F_DICT_SIZE); + if (preferencesPtr->autoFlush) + requiredBuffSize = (cctxPtr->prefs.frameInfo.blockMode == LZ5F_blockLinked) * LZ5F_DICT_SIZE; /* just needs dict */ + + if (cctxPtr->maxBufferSize < requiredBuffSize) + { + cctxPtr->maxBufferSize = requiredBuffSize; + FREEMEM(cctxPtr->tmpBuff); + cctxPtr->tmpBuff = (BYTE*)ALLOCATOR(requiredBuffSize); + if (cctxPtr->tmpBuff == NULL) return (size_t)-LZ5F_ERROR_allocation_failed; + } + cctxPtr->tmpIn = cctxPtr->tmpBuff; + cctxPtr->tmpInSize = 0; + XXH32_reset(&(cctxPtr->xxh), 0); + if (cctxPtr->prefs.compressionLevel < minHClevel) + LZ5_resetStream((LZ5_stream_t*)(cctxPtr->lz5CtxPtr)); + else + LZ5_resetStreamHC((LZ5_streamHC_t*)(cctxPtr->lz5CtxPtr)); + + /* Magic Number */ + LZ5F_writeLE32(dstPtr, LZ5F_MAGICNUMBER); + dstPtr += 4; + headerStart = dstPtr; + + /* FLG Byte */ + *dstPtr++ = (BYTE)(((1 & _2BITS) << 6) /* Version('01') */ + + ((cctxPtr->prefs.frameInfo.blockMode & _1BIT ) << 5) /* Block mode */ + + ((cctxPtr->prefs.frameInfo.contentChecksumFlag & _1BIT ) << 2) /* Frame checksum */ + + ((cctxPtr->prefs.frameInfo.contentSize > 0) << 3)); /* Frame content size */ + /* BD Byte */ + *dstPtr++ = (BYTE)((cctxPtr->prefs.frameInfo.blockSizeID & _3BITS) << 4); + /* Optional Frame content size field */ + if (cctxPtr->prefs.frameInfo.contentSize) + { + LZ5F_writeLE64(dstPtr, cctxPtr->prefs.frameInfo.contentSize); + dstPtr += 8; + cctxPtr->totalInSize = 0; + } + /* CRC Byte */ + *dstPtr = LZ5F_headerChecksum(headerStart, dstPtr - headerStart); + dstPtr++; + + cctxPtr->cStage = 1; /* header written, now request input data block */ + + return (dstPtr - dstStart); +} + + +/* LZ5F_compressBound() : gives the size of Dst buffer given a srcSize to handle worst case situations. +* The LZ5F_frameInfo_t structure is optional : +* you can provide NULL as argument, preferences will then be set to cover worst case situations. +* */ +size_t LZ5F_compressBound(size_t srcSize, const LZ5F_preferences_t* preferencesPtr) +{ + LZ5F_preferences_t prefsNull; + memset(&prefsNull, 0, sizeof(prefsNull)); + prefsNull.frameInfo.contentChecksumFlag = LZ5F_contentChecksumEnabled; /* worst case */ + { + const LZ5F_preferences_t* prefsPtr = (preferencesPtr==NULL) ? &prefsNull : preferencesPtr; + LZ5F_blockSizeID_t bid = prefsPtr->frameInfo.blockSizeID; + size_t blockSize = LZ5F_getBlockSize(bid); + unsigned nbBlocks = (unsigned)(srcSize / blockSize) + 1; + size_t lastBlockSize = prefsPtr->autoFlush ? srcSize % blockSize : blockSize; + size_t blockInfo = 4; /* default, without block CRC option */ + size_t frameEnd = 4 + (prefsPtr->frameInfo.contentChecksumFlag*4); + + return (blockInfo * nbBlocks) + (blockSize * (nbBlocks-1)) + lastBlockSize + frameEnd;; + } +} + + +typedef int (*compressFunc_t)(void* ctx, const char* src, char* dst, int srcSize, int dstSize); + +static size_t LZ5F_compressBlock(void* dst, const void* src, size_t srcSize, compressFunc_t compress, void* lz5ctx) +{ + /* compress one block */ + BYTE* cSizePtr = (BYTE*)dst; + U32 cSize; + cSize = (U32)compress(lz5ctx, (const char*)src, (char*)(cSizePtr+4), (int)(srcSize), (int)(srcSize-1)); + LZ5F_writeLE32(cSizePtr, cSize); + if (cSize == 0) /* compression failed */ + { + cSize = (U32)srcSize; + LZ5F_writeLE32(cSizePtr, cSize + LZ5F_BLOCKUNCOMPRESSED_FLAG); + memcpy(cSizePtr+4, src, srcSize); + } + return cSize + 4; +} + + +static int LZ5F_localLZ5_compress_limitedOutput_withState(void* ctx, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ5_compress_limitedOutput_withState(ctx, src, dst, srcSize, dstSize); +} + +static int LZ5F_localLZ5_compress_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ5_compress_limitedOutput_continue((LZ5_stream_t*)ctx, src, dst, srcSize, dstSize); +} + +static int LZ5F_localLZ5_compressHC_limitedOutput_continue(void* ctx, const char* src, char* dst, int srcSize, int dstSize) +{ + return LZ5_compress_HC_continue((LZ5_streamHC_t*)ctx, src, dst, srcSize, dstSize); +} + +static compressFunc_t LZ5F_selectCompression(LZ5F_blockMode_t blockMode, int level) +{ + if (level < minHClevel) + { + if (blockMode == LZ5F_blockIndependent) return LZ5F_localLZ5_compress_limitedOutput_withState; + return LZ5F_localLZ5_compress_limitedOutput_continue; + } + if (blockMode == LZ5F_blockIndependent) return LZ5_compress_HC_extStateHC; + return LZ5F_localLZ5_compressHC_limitedOutput_continue; +} + +static int LZ5F_localSaveDict(LZ5F_cctx_t* cctxPtr) +{ + if (cctxPtr->prefs.compressionLevel < minHClevel) + return LZ5_saveDict ((LZ5_stream_t*)(cctxPtr->lz5CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); + return LZ5_saveDictHC ((LZ5_streamHC_t*)(cctxPtr->lz5CtxPtr), (char*)(cctxPtr->tmpBuff), 64 KB); +} + +typedef enum { notDone, fromTmpBuffer, fromSrcBuffer } LZ5F_lastBlockStatus; + +/* LZ5F_compressUpdate() +* LZ5F_compressUpdate() can be called repetitively to compress as much data as necessary. +* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. +* If this condition is not respected, LZ5F_compress() will fail (result is an errorCode) +* You can get the minimum value of dstMaxSize by using LZ5F_compressBound() +* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument. +* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. +* The function outputs an error code if it fails (can be tested using LZ5F_isError()) +*/ +size_t LZ5F_compressUpdate(LZ5F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_compressOptions_t* compressOptionsPtr) +{ + LZ5F_compressOptions_t cOptionsNull; + LZ5F_cctx_t* cctxPtr = (LZ5F_cctx_t*)compressionContext; + size_t blockSize = cctxPtr->maxBlockSize; + const BYTE* srcPtr = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcPtr + srcSize; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + LZ5F_lastBlockStatus lastBlockCompressed = notDone; + compressFunc_t compress; + + + if (cctxPtr->cStage != 1) return (size_t)-LZ5F_ERROR_GENERIC; + if (dstMaxSize < LZ5F_compressBound(srcSize, &(cctxPtr->prefs))) return (size_t)-LZ5F_ERROR_dstMaxSize_tooSmall; + memset(&cOptionsNull, 0, sizeof(cOptionsNull)); + if (compressOptionsPtr == NULL) compressOptionsPtr = &cOptionsNull; + + /* select compression function */ + compress = LZ5F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + + /* complete tmp buffer */ + if (cctxPtr->tmpInSize > 0) /* some data already within tmp buffer */ + { + size_t sizeToCopy = blockSize - cctxPtr->tmpInSize; + if (sizeToCopy > srcSize) + { + /* add src to tmpIn buffer */ + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, srcSize); + srcPtr = srcEnd; + cctxPtr->tmpInSize += srcSize; + /* still needs some CRC */ + } + else + { + /* complete tmpIn block and then compress it */ + lastBlockCompressed = fromTmpBuffer; + memcpy(cctxPtr->tmpIn + cctxPtr->tmpInSize, srcBuffer, sizeToCopy); + srcPtr += sizeToCopy; + + dstPtr += LZ5F_compressBlock(dstPtr, cctxPtr->tmpIn, blockSize, compress, cctxPtr->lz5CtxPtr); + + if (cctxPtr->prefs.frameInfo.blockMode==LZ5F_blockLinked) cctxPtr->tmpIn += blockSize; + cctxPtr->tmpInSize = 0; + } + } + + while ((size_t)(srcEnd - srcPtr) >= blockSize) + { + /* compress full block */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ5F_compressBlock(dstPtr, srcPtr, blockSize, compress, cctxPtr->lz5CtxPtr); + srcPtr += blockSize; + } + + if ((cctxPtr->prefs.autoFlush) && (srcPtr < srcEnd)) + { + /* compress remaining input < blockSize */ + lastBlockCompressed = fromSrcBuffer; + dstPtr += LZ5F_compressBlock(dstPtr, srcPtr, srcEnd - srcPtr, compress, cctxPtr->lz5CtxPtr); + srcPtr = srcEnd; + } + + /* preserve dictionary if necessary */ + if ((cctxPtr->prefs.frameInfo.blockMode==LZ5F_blockLinked) && (lastBlockCompressed==fromSrcBuffer)) + { + if (compressOptionsPtr->stableSrc) + { + cctxPtr->tmpIn = cctxPtr->tmpBuff; + } + else + { + int realDictSize = LZ5F_localSaveDict(cctxPtr); + if (realDictSize==0) return (size_t)-LZ5F_ERROR_GENERIC; + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + } + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + blockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize) /* necessarily LZ5F_blockLinked && lastBlockCompressed==fromTmpBuffer */ + && !(cctxPtr->prefs.autoFlush)) + { + int realDictSize = LZ5F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + /* some input data left, necessarily < blockSize */ + if (srcPtr < srcEnd) + { + /* fill tmp buffer */ + size_t sizeToCopy = srcEnd - srcPtr; + memcpy(cctxPtr->tmpIn, srcPtr, sizeToCopy); + cctxPtr->tmpInSize = sizeToCopy; + } + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ5F_contentChecksumEnabled) + XXH32_update(&(cctxPtr->xxh), srcBuffer, srcSize); + + cctxPtr->totalInSize += srcSize; + return dstPtr - dstStart; +} + + +/* LZ5F_flush() +* Should you need to create compressed data immediately, without waiting for a block to be filled, +* you can call LZ5_flush(), which will immediately compress any remaining data stored within compressionContext. +* The result of the function is the number of bytes written into dstBuffer +* (it can be zero, this means there was no data left within compressionContext) +* The function outputs an error code if it fails (can be tested using LZ5F_isError()) +* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument. +*/ +size_t LZ5F_flush(LZ5F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* compressOptionsPtr) +{ + LZ5F_cctx_t* cctxPtr = (LZ5F_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + compressFunc_t compress; + + + if (cctxPtr->tmpInSize == 0) return 0; /* nothing to flush */ + if (cctxPtr->cStage != 1) return (size_t)-LZ5F_ERROR_GENERIC; + if (dstMaxSize < (cctxPtr->tmpInSize + 8)) return (size_t)-LZ5F_ERROR_dstMaxSize_tooSmall; /* +8 : block header(4) + block checksum(4) */ + (void)compressOptionsPtr; /* not yet useful */ + + /* select compression function */ + compress = LZ5F_selectCompression(cctxPtr->prefs.frameInfo.blockMode, cctxPtr->prefs.compressionLevel); + + /* compress tmp buffer */ + dstPtr += LZ5F_compressBlock(dstPtr, cctxPtr->tmpIn, cctxPtr->tmpInSize, compress, cctxPtr->lz5CtxPtr); + if (cctxPtr->prefs.frameInfo.blockMode==LZ5F_blockLinked) cctxPtr->tmpIn += cctxPtr->tmpInSize; + cctxPtr->tmpInSize = 0; + + /* keep tmpIn within limits */ + if ((cctxPtr->tmpIn + cctxPtr->maxBlockSize) > (cctxPtr->tmpBuff + cctxPtr->maxBufferSize)) /* necessarily LZ5F_blockLinked */ + { + int realDictSize = LZ5F_localSaveDict(cctxPtr); + cctxPtr->tmpIn = cctxPtr->tmpBuff + realDictSize; + } + + return dstPtr - dstStart; +} + + +/* LZ5F_compressEnd() +* When you want to properly finish the compressed frame, just call LZ5F_compressEnd(). +* It will flush whatever data remained within compressionContext (like LZ5_flush()) +* but also properly finalize the frame, with an endMark and a checksum. +* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark size)) +* The function outputs an error code if it fails (can be tested using LZ5F_isError()) +* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument. +* compressionContext can then be used again, starting with LZ5F_compressBegin(). The preferences will remain the same. +*/ +size_t LZ5F_compressEnd(LZ5F_compressionContext_t compressionContext, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* compressOptionsPtr) +{ + LZ5F_cctx_t* cctxPtr = (LZ5F_cctx_t*)compressionContext; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* dstPtr = dstStart; + size_t errorCode; + + errorCode = LZ5F_flush(compressionContext, dstBuffer, dstMaxSize, compressOptionsPtr); + if (LZ5F_isError(errorCode)) return errorCode; + dstPtr += errorCode; + + LZ5F_writeLE32(dstPtr, 0); + dstPtr+=4; /* endMark */ + + if (cctxPtr->prefs.frameInfo.contentChecksumFlag == LZ5F_contentChecksumEnabled) + { + U32 xxh = XXH32_digest(&(cctxPtr->xxh)); + LZ5F_writeLE32(dstPtr, xxh); + dstPtr+=4; /* content Checksum */ + } + + cctxPtr->cStage = 0; /* state is now re-usable (with identical preferences) */ + + if (cctxPtr->prefs.frameInfo.contentSize) + { + if (cctxPtr->prefs.frameInfo.contentSize != cctxPtr->totalInSize) + return (size_t)-LZ5F_ERROR_frameSize_wrong; + } + + return dstPtr - dstStart; +} + + +/********************************** +* Decompression functions +**********************************/ + +/* Resource management */ + +/* LZ5F_createDecompressionContext() : +* The first thing to do is to create a decompressionContext object, which will be used in all decompression operations. +* This is achieved using LZ5F_createDecompressionContext(). +* The function will provide a pointer to a fully allocated and initialized LZ5F_decompressionContext object. +* If the result LZ5F_errorCode_t is not zero, there was an error during context creation. +* Object can release its memory using LZ5F_freeDecompressionContext(); +*/ +LZ5F_errorCode_t LZ5F_createDecompressionContext(LZ5F_decompressionContext_t* LZ5F_decompressionContextPtr, unsigned versionNumber) +{ + LZ5F_dctx_t* dctxPtr; + + dctxPtr = (LZ5F_dctx_t*)ALLOCATOR(sizeof(LZ5F_dctx_t)); + if (dctxPtr==NULL) return (LZ5F_errorCode_t)-LZ5F_ERROR_GENERIC; + + dctxPtr->version = versionNumber; + *LZ5F_decompressionContextPtr = (LZ5F_decompressionContext_t)dctxPtr; + return LZ5F_OK_NoError; +} + +LZ5F_errorCode_t LZ5F_freeDecompressionContext(LZ5F_decompressionContext_t LZ5F_decompressionContext) +{ + LZ5F_errorCode_t result = LZ5F_OK_NoError; + LZ5F_dctx_t* dctxPtr = (LZ5F_dctx_t*)LZ5F_decompressionContext; + if (dctxPtr != NULL) /* can accept NULL input, like free() */ + { + result = (LZ5F_errorCode_t)dctxPtr->dStage; + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + FREEMEM(dctxPtr); + } + return result; +} + + +/* ******************************************************************** */ +/* ********************* Decompression ******************************** */ +/* ******************************************************************** */ + +typedef enum { dstage_getHeader=0, dstage_storeHeader, + dstage_getCBlockSize, dstage_storeCBlockSize, + dstage_copyDirect, + dstage_getCBlock, dstage_storeCBlock, + dstage_decodeCBlock, dstage_decodeCBlock_intoDst, + dstage_decodeCBlock_intoTmp, dstage_flushOut, + dstage_getSuffix, dstage_storeSuffix, + dstage_getSFrameSize, dstage_storeSFrameSize, + dstage_skipSkippable +} dStage_t; + + +/* LZ5F_decodeHeader + return : nb Bytes read from srcVoidPtr (necessarily <= srcSize) + or an error code (testable with LZ5F_isError()) + output : set internal values of dctx, such as + dctxPtr->frameInfo and dctxPtr->dStage. + input : srcVoidPtr points at the **beginning of the frame** +*/ +static size_t LZ5F_decodeHeader(LZ5F_dctx_t* dctxPtr, const void* srcVoidPtr, size_t srcSize) +{ + BYTE FLG, BD, HC; + unsigned version, blockMode, blockChecksumFlag, contentSizeFlag, contentChecksumFlag, blockSizeID; + size_t bufferNeeded; + size_t frameHeaderSize; + const BYTE* srcPtr = (const BYTE*)srcVoidPtr; + + /* need to decode header to get frameInfo */ + if (srcSize < minFHSize) return (size_t)-LZ5F_ERROR_frameHeader_incomplete; /* minimal frame header size */ + memset(&(dctxPtr->frameInfo), 0, sizeof(dctxPtr->frameInfo)); + + /* special case : skippable frames */ + if ((LZ5F_readLE32(srcPtr) & 0xFFFFFFF0U) == LZ5F_MAGIC_SKIPPABLE_START) + { + dctxPtr->frameInfo.frameType = LZ5F_skippableFrame; + if (srcVoidPtr == (void*)(dctxPtr->header)) + { + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + return srcSize; + } + else + { + dctxPtr->dStage = dstage_getSFrameSize; + return 4; + } + } + + /* control magic number */ + if (LZ5F_readLE32(srcPtr) != LZ5F_MAGICNUMBER) return (size_t)-LZ5F_ERROR_frameType_unknown; + dctxPtr->frameInfo.frameType = LZ5F_frame; + + /* Flags */ + FLG = srcPtr[4]; + version = (FLG>>6) & _2BITS; + blockMode = (FLG>>5) & _1BIT; + blockChecksumFlag = (FLG>>4) & _1BIT; + contentSizeFlag = (FLG>>3) & _1BIT; + contentChecksumFlag = (FLG>>2) & _1BIT; + + /* Frame Header Size */ + frameHeaderSize = contentSizeFlag ? maxFHSize : minFHSize; + + if (srcSize < frameHeaderSize) + { + /* not enough input to fully decode frame header */ + if (srcPtr != dctxPtr->header) + memcpy(dctxPtr->header, srcPtr, srcSize); + dctxPtr->tmpInSize = srcSize; + dctxPtr->tmpInTarget = frameHeaderSize; + dctxPtr->dStage = dstage_storeHeader; + return srcSize; + } + + BD = srcPtr[5]; + blockSizeID = (BD>>4) & _3BITS; + + /* validate */ + if (version != 1) return (size_t)-LZ5F_ERROR_headerVersion_wrong; /* Version Number, only supported value */ + if (blockChecksumFlag != 0) return (size_t)-LZ5F_ERROR_blockChecksum_unsupported; /* Not supported for the time being */ + if (((FLG>>0)&_2BITS) != 0) return (size_t)-LZ5F_ERROR_reservedFlag_set; /* Reserved bits */ + if (((BD>>7)&_1BIT) != 0) return (size_t)-LZ5F_ERROR_reservedFlag_set; /* Reserved bit */ + if (blockSizeID < 1) return (size_t)-LZ5F_ERROR_maxBlockSize_invalid; /* 1-7 only supported values for the time being */ + if (((BD>>0)&_4BITS) != 0) return (size_t)-LZ5F_ERROR_reservedFlag_set; /* Reserved bits */ + + /* check */ + HC = LZ5F_headerChecksum(srcPtr+4, frameHeaderSize-5); + if (HC != srcPtr[frameHeaderSize-1]) return (size_t)-LZ5F_ERROR_headerChecksum_invalid; /* Bad header checksum error */ + + /* save */ + dctxPtr->frameInfo.blockMode = (LZ5F_blockMode_t)blockMode; + dctxPtr->frameInfo.contentChecksumFlag = (LZ5F_contentChecksum_t)contentChecksumFlag; + dctxPtr->frameInfo.blockSizeID = (LZ5F_blockSizeID_t)blockSizeID; + dctxPtr->maxBlockSize = LZ5F_getBlockSize(blockSizeID); + if (contentSizeFlag) + dctxPtr->frameRemainingSize = dctxPtr->frameInfo.contentSize = LZ5F_readLE64(srcPtr+6); + + /* init */ + if (contentChecksumFlag) XXH32_reset(&(dctxPtr->xxh), 0); + + /* alloc */ + bufferNeeded = dctxPtr->maxBlockSize + ((dctxPtr->frameInfo.blockMode==LZ5F_blockLinked) * 2 * LZ5F_DICT_SIZE); + if (bufferNeeded > dctxPtr->maxBufferSize) /* tmp buffers too small */ + { + FREEMEM(dctxPtr->tmpIn); + FREEMEM(dctxPtr->tmpOutBuffer); + dctxPtr->maxBufferSize = bufferNeeded; + dctxPtr->tmpIn = (BYTE*)ALLOCATOR(dctxPtr->maxBlockSize); + if (dctxPtr->tmpIn == NULL) return (size_t)-LZ5F_ERROR_GENERIC; + dctxPtr->tmpOutBuffer= (BYTE*)ALLOCATOR(dctxPtr->maxBufferSize); + if (dctxPtr->tmpOutBuffer== NULL) return (size_t)-LZ5F_ERROR_GENERIC; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = 0; + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = 0; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer; + dctxPtr->tmpOutStart = 0; + dctxPtr->tmpOutSize = 0; + + dctxPtr->dStage = dstage_getCBlockSize; + + return frameHeaderSize; +} + + +/* LZ5F_getFrameInfo() +* This function decodes frame header information, such as blockSize. +* It is optional : you could start by calling directly LZ5F_decompress() instead. +* The objective is to extract header information without starting decompression, typically for allocation purposes. +* LZ5F_getFrameInfo() can also be used *after* starting decompression, on a valid LZ5F_decompressionContext_t. +* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr) +* The function result is an hint of the better srcSize to use for next call to LZ5F_decompress, +* or an error code which can be tested using LZ5F_isError(). +*/ +LZ5F_errorCode_t LZ5F_getFrameInfo(LZ5F_decompressionContext_t dCtx, LZ5F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr) +{ + LZ5F_dctx_t* dctxPtr = (LZ5F_dctx_t*)dCtx; + + if (dctxPtr->dStage > dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ + { + size_t o=0, i=0; + /* frameInfo already decoded */ + *srcSizePtr = 0; + *frameInfoPtr = dctxPtr->frameInfo; + return LZ5F_decompress(dCtx, NULL, &o, NULL, &i, NULL); + } + else + { + size_t o=0; + size_t nextSrcSize = LZ5F_decompress(dCtx, NULL, &o, srcBuffer, srcSizePtr, NULL); + if (dctxPtr->dStage <= dstage_storeHeader) /* note : requires dstage_* header related to be at beginning of enum */ + return (size_t)-LZ5F_ERROR_frameHeader_incomplete; + *frameInfoPtr = dctxPtr->frameInfo; + return nextSrcSize; + } +} + + +/* trivial redirector, for common prototype */ +static int LZ5F_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize) +{ + (void)dictStart; (void)dictSize; + return LZ5_decompress_safe (source, dest, compressedSize, maxDecompressedSize); +} + + +static void LZ5F_updateDict(LZ5F_dctx_t* dctxPtr, const BYTE* dstPtr, size_t dstSize, const BYTE* dstPtr0, unsigned withinTmp) +{ + if (dctxPtr->dictSize==0) + dctxPtr->dict = (const BYTE*)dstPtr; /* priority to dictionary continuity */ + + if (dctxPtr->dict + dctxPtr->dictSize == dstPtr) /* dictionary continuity */ + { + dctxPtr->dictSize += dstSize; + return; + } + + if (dstPtr - dstPtr0 + dstSize >= 64 KB) /* dstBuffer large enough to become dictionary */ + { + dctxPtr->dict = (const BYTE*)dstPtr0; + dctxPtr->dictSize = dstPtr - dstPtr0 + dstSize; + return; + } + + if ((withinTmp) && (dctxPtr->dict == dctxPtr->tmpOutBuffer)) + { + /* assumption : dctxPtr->dict + dctxPtr->dictSize == dctxPtr->tmpOut + dctxPtr->tmpOutStart */ + dctxPtr->dictSize += dstSize; + return; + } + + if (withinTmp) /* copy relevant dict portion in front of tmpOut within tmpOutBuffer */ + { + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = 64 KB - dctxPtr->tmpOutSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart + dstSize; + return; + } + + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) /* copy dst into tmp to complete dict */ + { + if (dctxPtr->dictSize + dstSize > dctxPtr->maxBufferSize) /* tmp buffer not large enough */ + { + size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + dctxPtr->dictSize = preserveSize; + } + memcpy(dctxPtr->tmpOutBuffer + dctxPtr->dictSize, dstPtr, dstSize); + dctxPtr->dictSize += dstSize; + return; + } + + /* join dict & dest into tmp */ + { + size_t preserveSize = 64 KB - dstSize; /* note : dstSize < 64 KB */ + if (preserveSize > dctxPtr->dictSize) preserveSize = dctxPtr->dictSize; + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - preserveSize, preserveSize); + memcpy(dctxPtr->tmpOutBuffer + preserveSize, dstPtr, dstSize); + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dstSize; + } +} + + + +/* LZ5F_decompress() +* Call this function repetitively to regenerate data compressed within srcBuffer. +* The function will attempt to decode *srcSizePtr from srcBuffer, into dstBuffer of maximum size *dstSizePtr. +* +* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). +* +* The number of bytes effectively read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). +* If the number of bytes read is < number of bytes provided, then the decompression operation is not complete. +* You will have to call it again, continuing from where it stopped. +* +* The function result is an hint of the better srcSize to use for next call to LZ5F_decompress. +* Basically, it's the size of the current (or remaining) compressed block + header of next block. +* Respecting the hint provides some boost to performance, since it allows less buffer shuffling. +* Note that this is just a hint, you can always provide any srcSize you want. +* When a frame is fully decoded, the function result will be 0. +* If decompression failed, function result is an error code which can be tested using LZ5F_isError(). +*/ +size_t LZ5F_decompress(LZ5F_decompressionContext_t decompressionContext, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ5F_decompressOptions_t* decompressOptionsPtr) +{ + LZ5F_dctx_t* dctxPtr = (LZ5F_dctx_t*)decompressionContext; + LZ5F_decompressOptions_t optionsNull; + const BYTE* const srcStart = (const BYTE*)srcBuffer; + const BYTE* const srcEnd = srcStart + *srcSizePtr; + const BYTE* srcPtr = srcStart; + BYTE* const dstStart = (BYTE*)dstBuffer; + BYTE* const dstEnd = dstStart + *dstSizePtr; + BYTE* dstPtr = dstStart; + const BYTE* selectedIn = NULL; + unsigned doAnotherStage = 1; + size_t nextSrcSizeHint = 1; + + + memset(&optionsNull, 0, sizeof(optionsNull)); + if (decompressOptionsPtr==NULL) decompressOptionsPtr = &optionsNull; + *srcSizePtr = 0; + *dstSizePtr = 0; + + /* expect to continue decoding src buffer where it left previously */ + if (dctxPtr->srcExpect != NULL) + { + if (srcStart != dctxPtr->srcExpect) return (size_t)-LZ5F_ERROR_srcPtr_wrong; + } + + /* programmed as a state machine */ + + while (doAnotherStage) + { + + switch(dctxPtr->dStage) + { + + case dstage_getHeader: + { + if ((size_t)(srcEnd-srcPtr) >= maxFHSize) /* enough to decode - shortcut */ + { + LZ5F_errorCode_t errorCode = LZ5F_decodeHeader(dctxPtr, srcPtr, srcEnd-srcPtr); + if (LZ5F_isError(errorCode)) return errorCode; + srcPtr += errorCode; + break; + } + dctxPtr->tmpInSize = 0; + dctxPtr->tmpInTarget = minFHSize; /* minimum to attempt decode */ + dctxPtr->dStage = dstage_storeHeader; + } + + case dstage_storeHeader: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) + { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; /* rest of header + nextBlockHeader */ + doAnotherStage = 0; /* not enough src data, ask for some more */ + break; + } + { + LZ5F_errorCode_t errorCode = LZ5F_decodeHeader(dctxPtr, dctxPtr->header, dctxPtr->tmpInTarget); + if (LZ5F_isError(errorCode)) return errorCode; + } + break; + } + + case dstage_getCBlockSize: + { + if ((size_t)(srcEnd - srcPtr) >= BHSize) + { + selectedIn = srcPtr; + srcPtr += BHSize; + } + else + { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlockSize; + } + } + + if (dctxPtr->dStage == dstage_storeCBlockSize) + case dstage_storeCBlockSize: + { + size_t sizeToCopy = BHSize - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < BHSize) /* not enough input to get full cBlockSize; wait for more */ + { + nextSrcSizeHint = BHSize - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->tmpIn; + } + + /* case dstage_decodeCBlockSize: */ /* no more direct access, to prevent scan-build warning */ + { + size_t nextCBlockSize = LZ5F_readLE32(selectedIn) & 0x7FFFFFFFU; + if (nextCBlockSize==0) /* frameEnd signal, no more CBlock */ + { + dctxPtr->dStage = dstage_getSuffix; + break; + } + if (nextCBlockSize > dctxPtr->maxBlockSize) return (size_t)-LZ5F_ERROR_GENERIC; /* invalid cBlockSize */ + dctxPtr->tmpInTarget = nextCBlockSize; + if (LZ5F_readLE32(selectedIn) & LZ5F_BLOCKUNCOMPRESSED_FLAG) + { + dctxPtr->dStage = dstage_copyDirect; + break; + } + dctxPtr->dStage = dstage_getCBlock; + if (dstPtr==dstEnd) + { + nextSrcSizeHint = nextCBlockSize + BHSize; + doAnotherStage = 0; + } + break; + } + + case dstage_copyDirect: /* uncompressed block */ + { + size_t sizeToCopy = dctxPtr->tmpInTarget; + if ((size_t)(srcEnd-srcPtr) < sizeToCopy) sizeToCopy = srcEnd - srcPtr; /* not enough input to read full block */ + if ((size_t)(dstEnd-dstPtr) < sizeToCopy) sizeToCopy = dstEnd - dstPtr; + memcpy(dstPtr, srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), srcPtr, sizeToCopy); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= sizeToCopy; + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LZ5F_blockLinked) + LZ5F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 0); + + srcPtr += sizeToCopy; + dstPtr += sizeToCopy; + if (sizeToCopy == dctxPtr->tmpInTarget) /* all copied */ + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + dctxPtr->tmpInTarget -= sizeToCopy; /* still need to copy more */ + nextSrcSizeHint = dctxPtr->tmpInTarget + BHSize; + doAnotherStage = 0; + break; + } + + case dstage_getCBlock: /* entry from dstage_decodeCBlockSize */ + { + if ((size_t)(srcEnd-srcPtr) < dctxPtr->tmpInTarget) + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeCBlock; + break; + } + selectedIn = srcPtr; + srcPtr += dctxPtr->tmpInTarget; + dctxPtr->dStage = dstage_decodeCBlock; + break; + } + + case dstage_storeCBlock: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd-srcPtr)) sizeToCopy = srcEnd-srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + dctxPtr->tmpInSize += sizeToCopy; + srcPtr += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* need more input */ + { + nextSrcSizeHint = (dctxPtr->tmpInTarget - dctxPtr->tmpInSize) + BHSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + dctxPtr->dStage = dstage_decodeCBlock; + break; + } + + case dstage_decodeCBlock: + { + if ((size_t)(dstEnd-dstPtr) < dctxPtr->maxBlockSize) /* not enough place into dst : decode into tmpOut */ + dctxPtr->dStage = dstage_decodeCBlock_intoTmp; + else + dctxPtr->dStage = dstage_decodeCBlock_intoDst; + break; + } + + case dstage_decodeCBlock_intoDst: + { + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == LZ5F_blockLinked) + decoder = LZ5_decompress_safe_usingDict; + else + decoder = LZ5F_decompress_safe; + + decodedSize = decoder((const char*)selectedIn, (char*)dstPtr, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-LZ5F_ERROR_GENERIC; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dstPtr, decodedSize); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LZ5F_blockLinked) + LZ5F_updateDict(dctxPtr, dstPtr, decodedSize, dstStart, 0); + + dstPtr += decodedSize; + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + + case dstage_decodeCBlock_intoTmp: + { + /* not enough place into dst : decode into tmpOut */ + int (*decoder)(const char*, char*, int, int, const char*, int); + int decodedSize; + + if (dctxPtr->frameInfo.blockMode == LZ5F_blockLinked) + decoder = LZ5_decompress_safe_usingDict; + else + decoder = LZ5F_decompress_safe; + + /* ensure enough place for tmpOut */ + if (dctxPtr->frameInfo.blockMode == LZ5F_blockLinked) + { + if (dctxPtr->dict == dctxPtr->tmpOutBuffer) + { + if (dctxPtr->dictSize > 128 KB) + { + memcpy(dctxPtr->tmpOutBuffer, dctxPtr->dict + dctxPtr->dictSize - 64 KB, 64 KB); + dctxPtr->dictSize = 64 KB; + } + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + dctxPtr->dictSize; + } + else /* dict not within tmp */ + { + size_t reservedDictSpace = dctxPtr->dictSize; + if (reservedDictSpace > 64 KB) reservedDictSpace = 64 KB; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + reservedDictSpace; + } + } + + /* Decode */ + decodedSize = decoder((const char*)selectedIn, (char*)dctxPtr->tmpOut, (int)dctxPtr->tmpInTarget, (int)dctxPtr->maxBlockSize, (const char*)dctxPtr->dict, (int)dctxPtr->dictSize); + if (decodedSize < 0) return (size_t)-LZ5F_ERROR_decompressionFailed; /* decompression failed */ + if (dctxPtr->frameInfo.contentChecksumFlag) XXH32_update(&(dctxPtr->xxh), dctxPtr->tmpOut, decodedSize); + if (dctxPtr->frameInfo.contentSize) dctxPtr->frameRemainingSize -= decodedSize; + dctxPtr->tmpOutSize = decodedSize; + dctxPtr->tmpOutStart = 0; + dctxPtr->dStage = dstage_flushOut; + break; + } + + case dstage_flushOut: /* flush decoded data from tmpOut to dstBuffer */ + { + size_t sizeToCopy = dctxPtr->tmpOutSize - dctxPtr->tmpOutStart; + if (sizeToCopy > (size_t)(dstEnd-dstPtr)) sizeToCopy = dstEnd-dstPtr; + memcpy(dstPtr, dctxPtr->tmpOut + dctxPtr->tmpOutStart, sizeToCopy); + + /* dictionary management */ + if (dctxPtr->frameInfo.blockMode==LZ5F_blockLinked) + LZ5F_updateDict(dctxPtr, dstPtr, sizeToCopy, dstStart, 1); + + dctxPtr->tmpOutStart += sizeToCopy; + dstPtr += sizeToCopy; + + /* end of flush ? */ + if (dctxPtr->tmpOutStart == dctxPtr->tmpOutSize) + { + dctxPtr->dStage = dstage_getCBlockSize; + break; + } + nextSrcSizeHint = BHSize; + doAnotherStage = 0; /* still some data to flush */ + break; + } + + case dstage_getSuffix: + { + size_t suffixSize = dctxPtr->frameInfo.contentChecksumFlag * 4; + if (dctxPtr->frameRemainingSize) return (size_t)-LZ5F_ERROR_frameSize_wrong; /* incorrect frame size decoded */ + if (suffixSize == 0) /* frame completed */ + { + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + if ((srcEnd - srcPtr) < 4) /* not enough size for entire CRC */ + { + dctxPtr->tmpInSize = 0; + dctxPtr->dStage = dstage_storeSuffix; + } + else + { + selectedIn = srcPtr; + srcPtr += 4; + } + } + + if (dctxPtr->dStage == dstage_storeSuffix) + case dstage_storeSuffix: + { + size_t sizeToCopy = 4 - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->tmpIn + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < 4) /* not enough input to read complete suffix */ + { + nextSrcSizeHint = 4 - dctxPtr->tmpInSize; + doAnotherStage=0; + break; + } + selectedIn = dctxPtr->tmpIn; + } + + /* case dstage_checkSuffix: */ /* no direct call, to avoid scan-build warning */ + { + U32 readCRC = LZ5F_readLE32(selectedIn); + U32 resultCRC = XXH32_digest(&(dctxPtr->xxh)); + if (readCRC != resultCRC) return (size_t)-LZ5F_ERROR_contentChecksum_invalid; + nextSrcSizeHint = 0; + dctxPtr->dStage = dstage_getHeader; + doAnotherStage = 0; + break; + } + + case dstage_getSFrameSize: + { + if ((srcEnd - srcPtr) >= 4) + { + selectedIn = srcPtr; + srcPtr += 4; + } + else + { + /* not enough input to read cBlockSize field */ + dctxPtr->tmpInSize = 4; + dctxPtr->tmpInTarget = 8; + dctxPtr->dStage = dstage_storeSFrameSize; + } + } + + if (dctxPtr->dStage == dstage_storeSFrameSize) + case dstage_storeSFrameSize: + { + size_t sizeToCopy = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + if (sizeToCopy > (size_t)(srcEnd - srcPtr)) sizeToCopy = srcEnd - srcPtr; + memcpy(dctxPtr->header + dctxPtr->tmpInSize, srcPtr, sizeToCopy); + srcPtr += sizeToCopy; + dctxPtr->tmpInSize += sizeToCopy; + if (dctxPtr->tmpInSize < dctxPtr->tmpInTarget) /* not enough input to get full sBlockSize; wait for more */ + { + nextSrcSizeHint = dctxPtr->tmpInTarget - dctxPtr->tmpInSize; + doAnotherStage = 0; + break; + } + selectedIn = dctxPtr->header + 4; + } + + /* case dstage_decodeSFrameSize: */ /* no direct access */ + { + size_t SFrameSize = LZ5F_readLE32(selectedIn); + dctxPtr->frameInfo.contentSize = SFrameSize; + dctxPtr->tmpInTarget = SFrameSize; + dctxPtr->dStage = dstage_skipSkippable; + break; + } + + case dstage_skipSkippable: + { + size_t skipSize = dctxPtr->tmpInTarget; + if (skipSize > (size_t)(srcEnd-srcPtr)) skipSize = srcEnd-srcPtr; + srcPtr += skipSize; + dctxPtr->tmpInTarget -= skipSize; + doAnotherStage = 0; + nextSrcSizeHint = dctxPtr->tmpInTarget; + if (nextSrcSizeHint) break; + dctxPtr->dStage = dstage_getHeader; + break; + } + } + } + + /* preserve dictionary within tmp if necessary */ + if ( (dctxPtr->frameInfo.blockMode==LZ5F_blockLinked) + &&(dctxPtr->dict != dctxPtr->tmpOutBuffer) + &&(!decompressOptionsPtr->stableDst) + &&((unsigned)(dctxPtr->dStage-1) < (unsigned)(dstage_getSuffix-1)) + ) + { + if (dctxPtr->dStage == dstage_flushOut) + { + size_t preserveSize = dctxPtr->tmpOut - dctxPtr->tmpOutBuffer; + size_t copySize = 64 KB - dctxPtr->tmpOutSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize - dctxPtr->tmpOutStart; + if (dctxPtr->tmpOutSize > 64 KB) copySize = 0; + if (copySize > preserveSize) copySize = preserveSize; + + memcpy(dctxPtr->tmpOutBuffer + preserveSize - copySize, oldDictEnd - copySize, copySize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = preserveSize + dctxPtr->tmpOutStart; + } + else + { + size_t newDictSize = dctxPtr->dictSize; + const BYTE* oldDictEnd = dctxPtr->dict + dctxPtr->dictSize; + if ((newDictSize) > 64 KB) newDictSize = 64 KB; + + memcpy(dctxPtr->tmpOutBuffer, oldDictEnd - newDictSize, newDictSize); + + dctxPtr->dict = dctxPtr->tmpOutBuffer; + dctxPtr->dictSize = newDictSize; + dctxPtr->tmpOut = dctxPtr->tmpOutBuffer + newDictSize; + } + } + + /* require function to be called again from position where it stopped */ + if (srcPtrsrcExpect = srcPtr; + else + dctxPtr->srcExpect = NULL; + + *srcSizePtr = (srcPtr - srcStart); + *dstSizePtr = (dstPtr - dstStart); + return nextSrcSizeHint; +} diff --git a/C/lz5/lz5frame.h b/C/lz5/lz5frame.h new file mode 100644 index 00000000..245a13f9 --- /dev/null +++ b/C/lz5/lz5frame.h @@ -0,0 +1,306 @@ +/* + LZ5 auto-framing library + Header File + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ5 source repository : https://github.com/inikep/lz5 + - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ + +/* LZ5F is a stand-alone API to create LZ5-compressed frames + * conformant with specification v1.5.1. + * All related operations, including memory management, are handled internally by the library. + * You don't need lz5.h when using lz5frame.h. + * */ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/************************************** +* Includes +**************************************/ +#include /* size_t */ + + +/************************************** +* Error management +**************************************/ +typedef size_t LZ5F_errorCode_t; + +unsigned LZ5F_isError(LZ5F_errorCode_t code); +const char* LZ5F_getErrorName(LZ5F_errorCode_t code); /* return error code string; useful for debugging */ + + +/************************************** +* Frame compression types +**************************************/ +//#define LZ5F_DISABLE_OBSOLETE_ENUMS +#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS +# define LZ5F_OBSOLETE_ENUM(x) ,x +#else +# define LZ5F_OBSOLETE_ENUM(x) +#endif + +typedef enum { + LZ5F_default=0, + LZ5F_max64KB=1, + LZ5F_max256KB=2, + LZ5F_max1MB=3, + LZ5F_max4MB=4, + LZ5F_max16MB=5, + LZ5F_max64MB=6, + LZ5F_max256MB=7 + LZ5F_OBSOLETE_ENUM(max64KB = LZ5F_max64KB) + LZ5F_OBSOLETE_ENUM(max256KB = LZ5F_max256KB) + LZ5F_OBSOLETE_ENUM(max1MB = LZ5F_max1MB) + LZ5F_OBSOLETE_ENUM(max4MB = LZ5F_max4MB) +} LZ5F_blockSizeID_t; + +typedef enum { + LZ5F_blockLinked=0, + LZ5F_blockIndependent + LZ5F_OBSOLETE_ENUM(blockLinked = LZ5F_blockLinked) + LZ5F_OBSOLETE_ENUM(blockIndependent = LZ5F_blockIndependent) +} LZ5F_blockMode_t; + +typedef enum { + LZ5F_noContentChecksum=0, + LZ5F_contentChecksumEnabled + LZ5F_OBSOLETE_ENUM(noContentChecksum = LZ5F_noContentChecksum) + LZ5F_OBSOLETE_ENUM(contentChecksumEnabled = LZ5F_contentChecksumEnabled) +} LZ5F_contentChecksum_t; + +typedef enum { + LZ5F_frame=0, + LZ5F_skippableFrame + LZ5F_OBSOLETE_ENUM(skippableFrame = LZ5F_skippableFrame) +} LZ5F_frameType_t; + +#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS +typedef LZ5F_blockSizeID_t blockSizeID_t; +typedef LZ5F_blockMode_t blockMode_t; +typedef LZ5F_frameType_t frameType_t; +typedef LZ5F_contentChecksum_t contentChecksum_t; +#endif + +typedef struct { + LZ5F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */ + LZ5F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */ + LZ5F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */ + LZ5F_frameType_t frameType; /* LZ5F_frame, skippableFrame ; 0 == default */ + unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */ + unsigned reserved[2]; /* must be zero for forward compatibility */ +} LZ5F_frameInfo_t; + +typedef struct { + LZ5F_frameInfo_t frameInfo; + int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */ + unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */ + unsigned reserved[4]; /* must be zero for forward compatibility */ +} LZ5F_preferences_t; + + +/*********************************** +* Simple compression function +***********************************/ +size_t LZ5F_compressFrameBound(size_t srcSize, const LZ5F_preferences_t* preferencesPtr); + +size_t LZ5F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_preferences_t* preferencesPtr); +/* LZ5F_compressFrame() + * Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.1 + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ5F_compressFrameBound() + * If this condition is not respected, LZ5F_compressFrame() will fail (result is an errorCode) + * The LZ5F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default. + * The result of the function is the number of bytes written into dstBuffer. + * The function outputs an error code if it fails (can be tested using LZ5F_isError()) + */ + + + +/********************************** +* Advanced compression functions +**********************************/ +typedef struct LZ5F_cctx_s* LZ5F_compressionContext_t; /* must be aligned on 8-bytes */ + +typedef struct { + unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ5F_compress(); avoid saving src content within tmp buffer as future dictionary */ + unsigned reserved[3]; +} LZ5F_compressOptions_t; + +/* Resource Management */ + +#define LZ5F_VERSION 100 +LZ5F_errorCode_t LZ5F_createCompressionContext(LZ5F_compressionContext_t* cctxPtr, unsigned version); +LZ5F_errorCode_t LZ5F_freeCompressionContext(LZ5F_compressionContext_t cctx); +/* LZ5F_createCompressionContext() : + * The first thing to do is to create a compressionContext object, which will be used in all compression operations. + * This is achieved using LZ5F_createCompressionContext(), which takes as argument a version and an LZ5F_preferences_t structure. + * The version provided MUST be LZ5F_VERSION. It is intended to track potential version differences between different binaries. + * The function will provide a pointer to a fully allocated LZ5F_compressionContext_t object. + * If the result LZ5F_errorCode_t is not zero, there was an error during context creation. + * Object can release its memory using LZ5F_freeCompressionContext(); + */ + + +/* Compression */ + +size_t LZ5F_compressBegin(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_preferences_t* prefsPtr); +/* LZ5F_compressBegin() : + * will write the frame header into dstBuffer. + * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes. + * The LZ5F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default. + * The result of the function is the number of bytes written into dstBuffer for the header + * or an error code (can be tested using LZ5F_isError()) + */ + +size_t LZ5F_compressBound(size_t srcSize, const LZ5F_preferences_t* prefsPtr); +/* LZ5F_compressBound() : + * Provides the minimum size of Dst buffer given srcSize to handle worst case situations. + * Different preferences can produce different results. + * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case. + * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled) + */ + +size_t LZ5F_compressUpdate(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_compressOptions_t* cOptPtr); +/* LZ5F_compressUpdate() + * LZ5F_compressUpdate() can be called repetitively to compress as much data as necessary. + * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case. + * You can get the minimum value of dstMaxSize by using LZ5F_compressBound(). + * If this condition is not respected, LZ5F_compress() will fail (result is an errorCode). + * LZ5F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs. + * The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered. + * The function outputs an error code if it fails (can be tested using LZ5F_isError()) + */ + +size_t LZ5F_flush(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr); +/* LZ5F_flush() + * Should you need to generate compressed data immediately, without waiting for the current block to be filled, + * you can call LZ5_flush(), which will immediately compress any remaining data buffered within cctx. + * Note that dstMaxSize must be large enough to ensure the operation will be successful. + * LZ5F_compressOptions_t structure is optional : you can provide NULL as argument. + * The result of the function is the number of bytes written into dstBuffer + * (it can be zero, this means there was no data left within cctx) + * The function outputs an error code if it fails (can be tested using LZ5F_isError()) + */ + +size_t LZ5F_compressEnd(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr); +/* LZ5F_compressEnd() + * When you want to properly finish the compressed frame, just call LZ5F_compressEnd(). + * It will flush whatever data remained within compressionContext (like LZ5_flush()) + * but also properly finalize the frame, with an endMark and a checksum. + * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled) + * The function outputs an error code if it fails (can be tested using LZ5F_isError()) + * The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument. + * A successful call to LZ5F_compressEnd() makes cctx available again for next compression task. + */ + + +/*********************************** +* Decompression functions +***********************************/ + +typedef struct LZ5F_dctx_s* LZ5F_decompressionContext_t; /* must be aligned on 8-bytes */ + +typedef struct { + unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */ + unsigned reserved[3]; +} LZ5F_decompressOptions_t; + + +/* Resource management */ + +LZ5F_errorCode_t LZ5F_createDecompressionContext(LZ5F_decompressionContext_t* dctxPtr, unsigned version); +LZ5F_errorCode_t LZ5F_freeDecompressionContext(LZ5F_decompressionContext_t dctx); +/* LZ5F_createDecompressionContext() : + * The first thing to do is to create an LZ5F_decompressionContext_t object, which will be used in all decompression operations. + * This is achieved using LZ5F_createDecompressionContext(). + * The version provided MUST be LZ5F_VERSION. It is intended to track potential breaking differences between different versions. + * The function will provide a pointer to a fully allocated and initialized LZ5F_decompressionContext_t object. + * The result is an errorCode, which can be tested using LZ5F_isError(). + * dctx memory can be released using LZ5F_freeDecompressionContext(); + * The result of LZ5F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. + * That is, it should be == 0 if decompression has been completed fully and correctly. + */ + + +/* Decompression */ + +size_t LZ5F_getFrameInfo(LZ5F_decompressionContext_t dctx, + LZ5F_frameInfo_t* frameInfoPtr, + const void* srcBuffer, size_t* srcSizePtr); +/* LZ5F_getFrameInfo() + * This function decodes frame header information (such as max blockSize, frame checksum, etc.). + * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes. + * A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that. + * The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value). + * (note that LZ5F_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough) + * Frame header info is *copied into* an already allocated LZ5F_frameInfo_t structure. + * The function result is an hint about how many srcSize bytes LZ5F_decompress() expects for next call, + * or an error code which can be tested using LZ5F_isError() + * (typically, when there is not enough src bytes to fully decode the frame header) + * Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr) + */ + +size_t LZ5F_decompress(LZ5F_decompressionContext_t dctx, + void* dstBuffer, size_t* dstSizePtr, + const void* srcBuffer, size_t* srcSizePtr, + const LZ5F_decompressOptions_t* dOptPtr); +/* LZ5F_decompress() + * Call this function repetitively to regenerate data compressed within srcBuffer. + * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr. + * + * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value). + * + * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value). + * If number of bytes read is < number of bytes provided, then decompression operation is not completed. + * It typically happens when dstBuffer is not large enough to contain all decoded data. + * LZ5F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr) + * The function will check this condition, and refuse to continue if it is not respected. + * + * dstBuffer is supposed to be flushed between each call to the function, since its content will be overwritten. + * dst arguments can be changed at will with each consecutive call to the function. + * + * The function result is an hint of how many srcSize bytes LZ5F_decompress() expects for next call. + * Schematically, it's the size of the current (or remaining) compressed block + header of next block. + * Respecting the hint provides some boost to performance, since it does skip intermediate buffers. + * This is just a hint, you can always provide any srcSize you want. + * When a frame is fully decoded, the function result will be 0 (no more data expected). + * If decompression failed, function result is an error code, which can be tested using LZ5F_isError(). + * + * After a frame is fully decoded, dctx can be used again to decompress another frame. + */ + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz5/lz5frame_static.h b/C/lz5/lz5frame_static.h new file mode 100644 index 00000000..0fb38bde --- /dev/null +++ b/C/lz5/lz5frame_static.h @@ -0,0 +1,81 @@ +/* + LZ5 auto-framing library + Header File for static linking only + Copyright (C) 2011-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ5 source repository : https://github.com/inikep/lz5 + - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ + +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + +/* lz5frame_static.h should be used solely in the context of static linking. + * It contains definitions which may still change overtime. + * Never use it in the context of DLL linking. + * */ + + +/************************************** +* Includes +**************************************/ +#include "lz5frame.h" + + +/************************************** + * Error management + * ************************************/ +#define LZ5F_LIST_ERRORS(ITEM) \ + ITEM(OK_NoError) ITEM(ERROR_GENERIC) \ + ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \ + ITEM(ERROR_compressionLevel_invalid) \ + ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \ + ITEM(ERROR_allocation_failed) \ + ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \ + ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \ + ITEM(ERROR_srcPtr_wrong) \ + ITEM(ERROR_decompressionFailed) \ + ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \ + ITEM(ERROR_maxCode) + +//#define LZ5F_DISABLE_OLD_ENUMS +#ifndef LZ5F_DISABLE_OLD_ENUMS +#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM, ENUM = LZ5F_##ENUM, +#else +#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM, +#endif +typedef enum { LZ5F_LIST_ERRORS(LZ5F_GENERATE_ENUM) } LZ5F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */ + + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz5/lz5hc.c b/C/lz5/lz5hc.c new file mode 100644 index 00000000..f1c65af8 --- /dev/null +++ b/C/lz5/lz5hc.c @@ -0,0 +1,1929 @@ +/* + LZ5 HC - High Compression Mode of LZ5 + Copyright (C) 2011-2015, Yann Collet. + Copyright (C) 2015, Przemyslaw Skibinski + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ5 source repository : https://github.com/inikep/lz5 + - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ + + + + +/* ************************************* +* Includes +***************************************/ +#define LZ5HC_INCLUDES +#include "lz5common.h" +#include "lz5.h" +#include "lz5hc.h" +#include +#include + + +/************************************** +* HC Compression +**************************************/ + + +int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* ctx, int compressionLevel) +{ + ctx->compressionLevel = compressionLevel; + if (compressionLevel > g_maxCompressionLevel) ctx->compressionLevel = g_maxCompressionLevel; + if (compressionLevel < 1) ctx->compressionLevel = LZ5HC_compressionLevel_default; + + ctx->params = LZ5HC_defaultParameters[ctx->compressionLevel]; + + ctx->hashTable = (U32*) malloc(sizeof(U32)*(((size_t)1 << ctx->params.hashLog3)+((size_t)1 << ctx->params.hashLog))); + if (!ctx->hashTable) + return 0; + + ctx->hashTable3 = ctx->hashTable + ((size_t)1 << ctx->params.hashLog); + + ctx->chainTable = (U32*) malloc(sizeof(U32)*((size_t)1 << ctx->params.contentLog)); + if (!ctx->chainTable) + { + FREEMEM(ctx->hashTable); + ctx->hashTable = NULL; + return 0; + } + + return 1; +} + +void LZ5_free_mem_HC(LZ5HC_Data_Structure* ctx) +{ + if (!ctx) return; + if (ctx->chainTable) FREEMEM(ctx->chainTable); + if (ctx->hashTable) FREEMEM(ctx->hashTable); + ctx->base = NULL; +} + +static void LZ5HC_init (LZ5HC_Data_Structure* ctx, const BYTE* start) +{ +#ifdef LZ5_RESET_MEM + MEM_INIT((void*)ctx->hashTable, 0, sizeof(U32)*((1 << ctx->params.hashLog) + (1 << ctx->params.hashLog3))); + if (ctx->params.strategy >= LZ5HC_lowest_price) + MEM_INIT(ctx->chainTable, 0x01, sizeof(U32)*(1 << ctx->params.contentLog)); +#else +#ifdef _DEBUG + int i, len = sizeof(U32)*((1 << ctx->params.hashLog) + (1 << ctx->params.hashLog3)); + unsigned char* bytes = (unsigned char*)ctx->hashTable; + srand(0); + for (i=0; inextToUpdate = (U32)((size_t)1 << ctx->params.windowLog); + ctx->base = start - ((size_t)1 << ctx->params.windowLog); + ctx->end = start; + ctx->dictBase = start - ((size_t)1 << ctx->params.windowLog); + ctx->dictLimit = (U32)((size_t)1 << ctx->params.windowLog); + ctx->lowLimit = (U32)((size_t)1 << ctx->params.windowLog); + ctx->last_off = 1; +} + + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void LZ5HC_BinTree_Insert(LZ5HC_Data_Structure* ctx, const BYTE* ip) +{ +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; + const BYTE* const base = ctx->base; + const U32 target = (U32)(ip - base); + U32 idx = ctx->nextToUpdate; + + while(idx < target) + { + HashTable3[LZ5HC_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; + idx++; + } + + ctx->nextToUpdate = target; +#endif +} + + +/* Update chains up to "end" (excluded) */ +FORCE_INLINE void LZ5HC_BinTree_InsertFull(LZ5HC_Data_Structure* ctx, const BYTE* end, const BYTE* iHighLimit) +{ + U32* chainTable = ctx->chainTable; + U32* HashTable = ctx->hashTable; +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; +#endif + + U32 idx = ctx->nextToUpdate; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(end - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > idx) ? ctx->lowLimit : idx - (maxDistance - 1); + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match, *ip; + int nbAttempts; + U32 *ptr0, *ptr1, *HashPos; + U32 matchIndex, delta0, delta1; + size_t mlt; + + + while(idx < current) + { + ip = base + idx; + if (ip + MINMATCH > iHighLimit) return; + + HashPos = &HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = *HashPos; +#if MINMATCH == 3 + HashTable3[LZ5HC_hash3Ptr(ip, ctx->params.hashLog3)] = idx; +#endif + + // check rest of matches + ptr0 = &chainTable[(idx*2+1) & contentMask]; + ptr1 = &chainTable[(idx*2) & contentMask]; + delta0 = delta1 = idx - matchIndex; + nbAttempts = ctx->params.searchNum; + *HashPos = idx; + + // while ((matchIndex >= dictLimit) && (matchIndex < idx) && (idx - matchIndex) < MAX_DISTANCE && nbAttempts) + while ((matchIndex < current) && (matchIndex < idx) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + mlt = 0; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (MEM_read24(match) == MEM_read24(ip)) + { + mlt = MINMATCH + MEM_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + + if (mlt > LZ5_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iHighLimit); + + if (mlt > LZ5_OPT_NUM) break; + } + } + + if (*(ip+mlt) < *(match+mlt)) + { + *ptr0 = delta0; + ptr0 = &chainTable[(matchIndex*2) & contentMask]; + if (*ptr0 == (U32)-1) break; + delta0 = *ptr0; + delta1 += delta0; + matchIndex -= delta0; + } + else + { + *ptr1 = delta1; + ptr1 = &chainTable[(matchIndex*2+1) & contentMask]; + if (*ptr1 == (U32)-1) break; + delta1 = *ptr1; + delta0 += delta1; + matchIndex -= delta1; + } + } + + *ptr0 = (U32)-1; + *ptr1 = (U32)-1; + + // LZ5_LOG_MATCH("%d: LZMAX_UPDATE_HASH_BINTREE hash=%d inp=%d,%d,%d,%d (%c%c%c%c)\n", (int)(inp-base), hash, inp[0], inp[1], inp[2], inp[3], inp[0], inp[1], inp[2], inp[3]); + + idx++; + } + + ctx->nextToUpdate = current; +} + + +/* Update chains up to ip (excluded) */ +FORCE_INLINE void LZ5HC_Insert (LZ5HC_Data_Structure* ctx, const BYTE* ip) +{ + U32* chainTable = ctx->chainTable; + U32* HashTable = ctx->hashTable; +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; +#endif + const BYTE* const base = ctx->base; + const U32 target = (U32)(ip - base); + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + U32 idx = ctx->nextToUpdate; + + while(idx < target) + { + size_t h = LZ5HC_hashPtr(base+idx, ctx->params.hashLog, ctx->params.searchLength); + chainTable[idx & contentMask] = (U32)(idx - HashTable[h]); +// if (chainTable[idx & contentMask] == 1) chainTable[idx & contentMask] = (U32)0x01010101; + HashTable[h] = idx; +#if MINMATCH == 3 + HashTable3[LZ5HC_hash3Ptr(base+idx, ctx->params.hashLog3)] = idx; +#endif + idx++; + } + + ctx->nextToUpdate = target; +} + + +FORCE_INLINE int LZ5HC_FindBestMatch (LZ5HC_Data_Structure* ctx, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + U32 matchIndex; + const BYTE* match; + int nbAttempts=ctx->params.searchNum; + size_t ml=0, mlt; + + matchIndex = HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + + match = ip - ctx->last_off; + if (MEM_read24(match) == MEM_read24(ip)) + { + ml = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + return (int)ml; + } + +#if MINMATCH == 3 + { + U32 matchIndex3 = ctx->hashTable3[LZ5HC_hash3Ptr(ip, ctx->params.hashLog3)]; + if (matchIndex3 < current && matchIndex3 >= lowLimit) + { + size_t offset = (size_t)current - matchIndex3; + if (offset < LZ5_SHORT_OFFSET_DISTANCE) + { + match = ip - offset; + if (match > base && MEM_read24(ip) == MEM_read24(match)) + { + ml = 3;//MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + } + } + } + } +#endif + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) + { + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (!ml || (mlt > ml && LZ5HC_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) +// if (mlt > ml && (LZ5_NORMAL_MATCH_COST(mlt - MINMATCH, (ip - match == ctx->last_off) ? 0 : (ip - match)) < LZ5_NORMAL_MATCH_COST(ml - MINMATCH, (ip - *matchpos == ctx->last_off) ? 0 : (ip - *matchpos)) + (LZ5_NORMAL_LIT_COST(mlt - ml)))) + { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iLimit); + if (!ml || (mlt > ml && LZ5HC_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) + // if (mlt > ml && (LZ5_NORMAL_MATCH_COST(mlt - MINMATCH, (ip - match == ctx->last_off) ? 0 : (ip - match)) < LZ5_NORMAL_MATCH_COST(ml - MINMATCH, (ip - *matchpos == ctx->last_off) ? 0 : (ip - *matchpos)) + (LZ5_NORMAL_LIT_COST(mlt - ml)))) + { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + matchIndex -= chainTable[matchIndex & contentMask]; + } + + return (int)ml; +} + + +FORCE_INLINE int LZ5HC_FindMatchFast (LZ5HC_Data_Structure* ctx, U32 matchIndex, U32 matchIndex3, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const BYTE* match; + size_t ml=0, mlt; + + match = ip - ctx->last_off; + if (MEM_read24(match) == MEM_read24(ip)) + { + ml = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + return (int)ml; + } + +#if MINMATCH == 3 + if (matchIndex3 < current && matchIndex3 >= lowLimit) + { + size_t offset = (size_t)current - matchIndex3; + if (offset < LZ5_SHORT_OFFSET_DISTANCE) + { + match = ip - offset; + if (match > base && MEM_read24(ip) == MEM_read24(match)) + { + ml = 3;//MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + } + } + } +#endif + + if ((matchIndex < current) && (matchIndex>=lowLimit)) + { + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) + { + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (!ml || (mlt > ml && LZ5HC_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) + // if (ml==0 || ((mlt > ml) && LZ5_NORMAL_MATCH_COST(mlt - MINMATCH, (ip - match == ctx->last_off) ? 0 : (ip - match)) < LZ5_NORMAL_MATCH_COST(ml - MINMATCH, (ip - *matchpos == ctx->last_off) ? 0 : (ip - *matchpos)) + (LZ5_NORMAL_LIT_COST(mlt - ml)))) + { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iLimit); + if (!ml || (mlt > ml && LZ5HC_better_price((ip - *matchpos), ml, (ip - match), mlt, ctx->last_off))) +// if (ml==0 || ((mlt > ml) && LZ5_NORMAL_MATCH_COST(mlt - MINMATCH, (ip - match == ctx->last_off) ? 0 : (ip - match)) < LZ5_NORMAL_MATCH_COST(ml - MINMATCH, (ip - *matchpos == ctx->last_off) ? 0 : (ip - *matchpos)) + (LZ5_NORMAL_LIT_COST(mlt - ml)))) + { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + } + + return (int)ml; +} + + +FORCE_INLINE int LZ5HC_FindMatchFaster (LZ5HC_Data_Structure* ctx, U32 matchIndex, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const BYTE* match; + size_t ml=0, mlt; + + match = ip - ctx->last_off; + if (MEM_read24(match) == MEM_read24(ip)) + { + ml = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + *matchpos = match; + return (int)ml; + } + + if (matchIndex < current && matchIndex >= lowLimit) + { + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) + { + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iLimit); + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + } + + return (int)ml; +} + + +FORCE_INLINE int LZ5HC_FindMatchFastest (LZ5HC_Data_Structure* ctx, U32 matchIndex, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + const BYTE** matchpos) +{ + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const U32 dictLimit = ctx->dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const BYTE* match; + size_t ml=0, mlt; + + if (matchIndex < current && matchIndex >= lowLimit) + { + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + if (*(match+ml) == *(ip+ml) && (MEM_read32(match) == MEM_read32(ip))) + { + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = match; } + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iLimit) vLimit = iLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iLimit); + if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */ + } + } + } + + return (int)ml; +} + + +FORCE_INLINE size_t LZ5HC_GetWiderMatch ( + LZ5HC_Data_Structure* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + size_t longest, + const BYTE** matchpos, + const BYTE** startpos) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + U32 matchIndex; + int nbAttempts = ctx->params.searchNum; + + + /* First Match */ + matchIndex = HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + + match = ip - ctx->last_off; + if (MEM_read24(match) == MEM_read24(ip)) + { + size_t mlt = MEM_count(ip+MINMATCH, match+MINMATCH, iHighLimit) + MINMATCH; + + int back = 0; + while ((ip+back>iLowLimit) && (match+back > lowPrefixPtr) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > longest) + { + *matchpos = match+back; + *startpos = ip+back; + longest = (int)mlt; + } + } + +#if MINMATCH == 3 + { + U32 matchIndex3 = ctx->hashTable3[LZ5HC_hash3Ptr(ip, ctx->params.hashLog3)]; + if (matchIndex3 < current && matchIndex3 >= lowLimit) + { + size_t offset = (size_t)current - matchIndex3; + if (offset < LZ5_SHORT_OFFSET_DISTANCE) + { + match = ip - offset; + if (match > base && MEM_read24(ip) == MEM_read24(match)) + { + size_t mlt = MEM_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + + int back = 0; + while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; + mlt -= back; + + if (!longest || (mlt > longest && LZ5HC_better_price((ip + back - *matchpos), longest, (ip - match), mlt, ctx->last_off))) + // if (!longest || (mlt > longest && LZ5_NORMAL_MATCH_COST(mlt - MINMATCH, (ip - match == ctx->last_off) ? 0 : (ip - match)) < LZ5_NORMAL_MATCH_COST(longest - MINMATCH, (ip+back - *matchpos == ctx->last_off) ? 0 : (ip+back - *matchpos)) + LZ5_NORMAL_LIT_COST(mlt - longest))) + { + *matchpos = match + back; + *startpos = ip + back; + longest = (int)mlt; + } + } + } + } + } +#endif + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if (MEM_read32(match) == MEM_read32(ip)) + { + size_t mlt = MINMATCH + MEM_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int back = 0; + + while ((ip+back>iLowLimit) + && (match+back > lowPrefixPtr) + && (ip[back-1] == match[back-1])) + back--; + + mlt -= back; + + if (!longest || (mlt > longest && LZ5HC_better_price((ip+back - *matchpos), longest, (ip - match), mlt, ctx->last_off))) + { + longest = (int)mlt; + *matchpos = match+back; + *startpos = ip+back; + } + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + if (mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; } + } + } + matchIndex -= chainTable[matchIndex & contentMask]; + } + + + return longest; +} + + +FORCE_INLINE int LZ5HC_GetAllMatches ( + LZ5HC_Data_Structure* ctx, + const BYTE* const ip, + const BYTE* const iLowLimit, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ5HC_match_t* matches) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + U32* const HashTable3 = ctx->hashTable3; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + U32 matchIndex; + int nbAttempts = ctx->params.searchNum; + // bool fullSearch = (ctx->params.fullSearch >= 2); + int mnum = 0; + U32* HashPos, *HashPos3; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = *HashPos; +#if MINMATCH == 3 + HashPos3 = &HashTable3[LZ5HC_hash3Ptr(ip, ctx->params.hashLog3)]; + + if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) + { + size_t offset = current - *HashPos3; + if (offset < LZ5_SHORT_OFFSET_DISTANCE) + { + match = ip - offset; + if (match > base && MEM_read24(ip) == MEM_read24(match)) + { + size_t mlt = MEM_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + + int back = 0; + while ((ip + back > iLowLimit) && (match + back > lowPrefixPtr) && (ip[back - 1] == match[back - 1])) back--; + mlt -= back; + + matches[mnum].off = (int)offset; + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + } + } + + *HashPos3 = current; +#endif + + + chainTable[current & contentMask] = (U32)(current - matchIndex); + *HashPos = current; + ctx->nextToUpdate++; + + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if ((/*fullSearch ||*/ ip[best_mlen] == match[best_mlen]) && (MEM_read24(match) == MEM_read24(ip))) + { + size_t mlt = MINMATCH + MEM_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + int back = 0; + + while ((ip+back>iLowLimit) + && (match+back > lowPrefixPtr) + && (ip[back-1] == match[back-1])) + back--; + + mlt -= back; + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + + if (best_mlen > LZ5_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + size_t mlt; + int back=0; + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iHighLimit); + while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == match[back-1])) back--; + mlt -= back; + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = -back; + mnum++; + } + + if (best_mlen > LZ5_OPT_NUM) break; + } + } + matchIndex -= chainTable[matchIndex & contentMask]; + } + + + return mnum; +} + + + +FORCE_INLINE int LZ5HC_BinTree_GetAllMatches ( + LZ5HC_Data_Structure* ctx, + const BYTE* const ip, + const BYTE* const iHighLimit, + size_t best_mlen, + LZ5HC_match_t* matches) +{ + U32* const chainTable = ctx->chainTable; + U32* const HashTable = ctx->hashTable; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 maxDistance = (1 << ctx->params.windowLog); + const U32 current = (U32)(ip - base); + const U32 lowLimit = (ctx->lowLimit + maxDistance > current) ? ctx->lowLimit : current - (maxDistance - 1); + const U32 contentMask = (1 << ctx->params.contentLog) - 1; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* match; + int nbAttempts = ctx->params.searchNum; + int mnum = 0; + U32 *ptr0, *ptr1; + U32 matchIndex, delta0, delta1; + size_t mlt = 0; + U32* HashPos, *HashPos3; + + if (ip + MINMATCH > iHighLimit) return 0; + + /* First Match */ + HashPos = &HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + matchIndex = *HashPos; + + +#if MINMATCH == 3 + HashPos3 = &ctx->hashTable3[LZ5HC_hash3Ptr(ip, ctx->params.hashLog3)]; + + if ((*HashPos3 < current) && (*HashPos3 >= lowLimit)) + { + size_t offset = current - *HashPos3; + if (offset < LZ5_SHORT_OFFSET_DISTANCE) + { + match = ip - offset; + if (match > base && MEM_read24(ip) == MEM_read24(match)) + { + mlt = MEM_count(ip + MINMATCH, match + MINMATCH, iHighLimit) + MINMATCH; + + matches[mnum].off = (int)offset; + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + } + + *HashPos3 = current; + } +#endif + + + *HashPos = current; + ctx->nextToUpdate++; + + // check rest of matches + ptr0 = &chainTable[(current*2+1) & contentMask]; + ptr1 = &chainTable[(current*2) & contentMask]; + delta0 = delta1 = current - matchIndex; + + while ((matchIndex < current) && (matchIndex>=lowLimit) && (nbAttempts)) + { + nbAttempts--; + mlt = 0; + if (matchIndex >= dictLimit) + { + match = base + matchIndex; + + if (MEM_read24(match) == MEM_read24(ip)) + { + mlt = MINMATCH + MEM_count(ip+MINMATCH, match+MINMATCH, iHighLimit); + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + + if (best_mlen > LZ5_OPT_NUM) break; + } + } + else + { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) + { + const BYTE* vLimit = ip + (dictLimit - matchIndex); + if (vLimit > iHighLimit) vLimit = iHighLimit; + mlt = MEM_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH; + if ((ip+mlt == vLimit) && (vLimit < iHighLimit)) + mlt += MEM_count(ip+mlt, base+dictLimit, iHighLimit); + + if (mlt > best_mlen) + { + best_mlen = mlt; + matches[mnum].off = (int)(ip - match); + matches[mnum].len = (int)mlt; + matches[mnum].back = 0; + mnum++; + } + + if (best_mlen > LZ5_OPT_NUM) break; + } + } + + if (*(ip+mlt) < *(match+mlt)) + { + *ptr0 = delta0; + ptr0 = &chainTable[(matchIndex*2) & contentMask]; + // printf("delta0=%d\n", delta0); + if (*ptr0 == (U32)-1) break; + delta0 = *ptr0; + delta1 += delta0; + matchIndex -= delta0; + } + else + { + *ptr1 = delta1; + ptr1 = &chainTable[(matchIndex*2+1) & contentMask]; + // printf("delta1=%d\n", delta1); + if (*ptr1 == (U32)-1) break; + delta1 = *ptr1; + delta0 += delta1; + matchIndex -= delta1; + } + } + + *ptr0 = (U32)-1; + *ptr1 = (U32)-1; + + return mnum; +} + + + + +typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive; + +/* +LZ5 uses 3 types of codewords from 2 to 4 bytes long: +- 1_OO_LL_MMM OOOOOOOO - 10-bit offset, 3-bit match length, 2-bit literal length +- 00_LLL_MMM OOOOOOOO OOOOOOOO - 16-bit offset, 3-bit match length, 3-bit literal length +- 010_LL_MMM OOOOOOOO OOOOOOOO OOOOOOOO - 24-bit offset, 3-bit match length, 2-bit literal length +- 011_LL_MMM - last offset, 3-bit match length, 2-bit literal length +*/ + +FORCE_INLINE int LZ5HC_encodeSequence ( + LZ5HC_Data_Structure* ctx, + const BYTE** ip, + BYTE** op, + const BYTE** anchor, + int matchLength, + const BYTE* const match, + limitedOutput_directive limitedOutputBuffer, + BYTE* oend) +{ + int length; + BYTE* token; + + /* Encode Literal length */ + length = (int)(*ip - *anchor); + token = (*op)++; + + if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */ + + if (*ip-match >= LZ5_SHORT_OFFSET_DISTANCE && *ip-match < LZ5_MID_OFFSET_DISTANCE && (U32)(*ip-match) != 0) + { + if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } + else *token = (BYTE)(length<=(int)RUN_MASK2) { int len; *token=(RUN_MASK2< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } + else *token = (BYTE)(length<last_off = (U32)(*ip-match); + if (ctx->last_off < LZ5_SHORT_OFFSET_DISTANCE) + { + *token+=(BYTE)((4+(ctx->last_off>>8))<last_off; (*op)++; + } + else + if (*ip-match < LZ5_MID_OFFSET_DISTANCE) + { + MEM_writeLE16(*op, (U16)ctx->last_off); *op+=2; + } + else + { + *token+=(2<last_off); *op+=3; + } + } + + /* Encode MatchLength */ + length = (int)(matchLength-MINMATCH); + if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */ + if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; } + else *token += (BYTE)(length); + + LZ5HC_DEBUG("%u: ENCODE literals=%u off=%u mlen=%u out=%u\n", (U32)(*ip - ctx->inputBuffer), (U32)(*ip - *anchor), (U32)(*ip-match), (U32)matchLength, 2+(U32)(*op - ctx->outputBuffer)); + + /* Prepare next loop */ + *ip += matchLength; + *anchor = *ip; + + return 0; +} + + +#define SET_PRICE(pos, mlen, offset, litlen, price) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = 1<<30; last_pos++; } \ + opt[pos].mlen = (int)mlen; \ + opt[pos].off = (int)offset; \ + opt[pos].litlen = (int)litlen; \ + opt[pos].price = (int)price; \ + LZ5_LOG_PARSER("%d: SET price[%d/%d]=%d litlen=%d len=%d off=%d\n", (int)(inr-source), pos, last_pos, opt[pos].price, opt[pos].litlen, opt[pos].mlen, opt[pos].off); \ + } + + +static int LZ5HC_compress_optimal_price ( + LZ5HC_Data_Structure* ctx, + const BYTE* source, + char* dest, + int inputSize, + int maxOutputSize, + limitedOutput_directive limit + ) +{ + LZ5HC_optimal_t opt[LZ5_OPT_NUM + 4]; + LZ5HC_match_t matches[LZ5_OPT_NUM + 1]; + const BYTE *inr; + size_t res, cur, cur2, skip_num = 0; + size_t i, llen, litlen, mlen, best_mlen, price, offset, best_off, match_num, last_pos; + + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + inputSize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + const size_t sufficient_len = ctx->params.sufficientLength; + const int faster_get_matches = (ctx->params.fullSearch == 0); + + + /* init */ + ctx->inputBuffer = (const BYTE*)source; + ctx->outputBuffer = (const BYTE*)dest; + ctx->end += inputSize; + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + memset(opt, 0, sizeof(LZ5HC_optimal_t)); + last_pos = 0; + llen = ip - anchor; + + // check rep + mlen = MEM_count(ip, ip - ctx->last_off, matchlimit); + if (mlen >= MINMATCH) + { + LZ5_LOG_PARSER("%d: start try REP rep=%d mlen=%d\n", (int)(ip-source), ctx->last_off, mlen); + if (mlen > sufficient_len || mlen >= LZ5_OPT_NUM) + { + best_mlen = mlen; best_off = 0; cur = 0; last_pos = 1; + goto encode; + } + + do + { + litlen = 0; + price = LZ5HC_get_price(llen, 0, mlen - MINMATCH) - llen; + if (mlen > last_pos || price < (size_t)opt[mlen].price) + SET_PRICE(mlen, mlen, 0, litlen, price); + mlen--; + } + while (mlen >= MINMATCH); + } + + + best_mlen = (last_pos) ? last_pos : MINMATCH; + + if (faster_get_matches && last_pos) + match_num = 0; + else + { + if (ctx->params.strategy == LZ5HC_optimal_price) + { + LZ5HC_Insert(ctx, ip); + match_num = LZ5HC_GetAllMatches(ctx, ip, ip, matchlimit, best_mlen, matches); + } + else + { + if (ctx->params.fullSearch < 2) + LZ5HC_BinTree_Insert(ctx, ip); + else + LZ5HC_BinTree_InsertFull(ctx, ip, matchlimit); + match_num = LZ5HC_BinTree_GetAllMatches(ctx, ip, matchlimit, best_mlen, matches); + } + } + + LZ5_LOG_PARSER("%d: match_num=%d last_pos=%d\n", (int)(ip-source), match_num, last_pos); + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (size_t)matches[match_num-1].len > sufficient_len) + { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto encode; + } + + // set prices using matches at position = 0 + for (i = 0; i < match_num; i++) + { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + best_mlen = (matches[i].len < LZ5_OPT_NUM) ? matches[i].len : LZ5_OPT_NUM; + LZ5_LOG_PARSER("%d: start Found mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(ip-source), matches[i].len, matches[i].off, best_mlen, last_pos); + while (mlen <= best_mlen) + { + litlen = 0; + price = LZ5HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + if (mlen > last_pos || price < (size_t)opt[mlen].price) + SET_PRICE(mlen, mlen, matches[i].off, litlen, price); + mlen++; + } + } + + if (last_pos < MINMATCH) { ip++; continue; } + + opt[0].rep = opt[1].rep = ctx->last_off; + opt[0].mlen = opt[1].mlen = 1; + + // check further positions + for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) + { + litlen = opt[cur-1].litlen + 1; + + if (cur != litlen) + { + price = opt[cur - litlen].price + LZ5_LIT_ONLY_COST(litlen); + LZ5_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + } + else + { + price = LZ5_LIT_ONLY_COST(llen + litlen) - llen; + LZ5_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + } + } + else + { + litlen = 1; + price = opt[cur - 1].price + LZ5_LIT_ONLY_COST(litlen); + LZ5_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d litonly=%d\n", (int)(inr-source), price, cur, litlen, LZ5_LIT_ONLY_COST(litlen)); + } + + mlen = 1; + best_mlen = 0; + LZ5_LOG_PARSER("%d: TRY price=%d opt[%d].price=%d\n", (int)(inr-source), price, cur, opt[cur].price); + + if (cur > last_pos || price <= (size_t)opt[cur].price) // || ((price == opt[cur].price) && (opt[cur-1].mlen == 1) && (cur != litlen))) + SET_PRICE(cur, mlen, best_mlen, litlen, price); + + if (cur == last_pos) break; + + if (opt[cur].mlen > 1) + { + mlen = opt[cur].mlen; + offset = opt[cur].off; + if (offset < 1) + { + opt[cur].rep = opt[cur-mlen].rep; + LZ5_LOG_PARSER("%d: COPYREP1 cur=%d mlen=%d rep=%d\n", (int)(inr-source), cur, mlen, opt[cur-mlen].rep); + } + else + { + opt[cur].rep = (int)offset; + LZ5_LOG_PARSER("%d: COPYREP2 cur=%d offset=%d rep=%d\n", (int)(inr-source), cur, offset, opt[cur].rep); + } + } + else + { + opt[cur].rep = opt[cur-1].rep; // copy rep + } + + + LZ5_LOG_PARSER("%d: CURRENT price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(inr-source), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); + + // check rep + // best_mlen = 0; + mlen = MEM_count(inr, inr - opt[cur].rep, matchlimit); + if (mlen >= MINMATCH && mlen > best_mlen) + { + LZ5_LOG_PARSER("%d: try REP rep=%d mlen=%d\n", (int)(inr-source), opt[cur].rep, mlen); + LZ5_LOG_PARSER("%d: Found REP mlen=%d off=%d rep=%d opt[%d].off=%d\n", (int)(inr-source), mlen, 0, opt[cur].rep, cur, opt[cur].off); + + if (mlen > sufficient_len || cur + mlen >= LZ5_OPT_NUM) + { + best_mlen = mlen; + best_off = 0; + LZ5_LOG_PARSER("%d: REP sufficient_len=%d best_mlen=%d best_off=%d last_pos=%d\n", (int)(inr-source), sufficient_len, best_mlen, best_off, last_pos); + last_pos = cur + 1; + goto encode; + } + + if (opt[cur].mlen == 1) + { + litlen = opt[cur].litlen; + + if (cur != litlen) + { + price = opt[cur - litlen].price + LZ5HC_get_price(litlen, 0, mlen - MINMATCH); + LZ5_LOG_PRICE("%d: TRY1 opt[%d].price=%d price=%d cur=%d litlen=%d\n", (int)(inr-source), cur - litlen, opt[cur - litlen].price, price, cur, litlen); + } + else + { + price = LZ5HC_get_price(llen + litlen, 0, mlen - MINMATCH) - llen; + LZ5_LOG_PRICE("%d: TRY2 price=%d cur=%d litlen=%d llen=%d\n", (int)(inr-source), price, cur, litlen, llen); + } + } + else + { + litlen = 0; + price = opt[cur].price + LZ5HC_get_price(litlen, 0, mlen - MINMATCH); + LZ5_LOG_PRICE("%d: TRY3 price=%d cur=%d litlen=%d getprice=%d\n", (int)(inr-source), price, cur, litlen, LZ5HC_get_price(litlen, 0, mlen - MINMATCH)); + } + + best_mlen = mlen; + if (faster_get_matches) + skip_num = best_mlen; + + LZ5_LOG_PARSER("%d: Found REP mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), mlen, 0, price, litlen, cur - litlen, opt[cur - litlen].price); + + do + { + if (cur + mlen > last_pos || price <= (size_t)opt[cur + mlen].price) // || ((price == opt[cur + mlen].price) && (opt[cur].mlen == 1) && (cur != litlen))) // at equal price prefer REP instead of MATCH + SET_PRICE(cur + mlen, mlen, 0, litlen, price); + mlen--; + } + while (mlen >= MINMATCH); + } + + if (faster_get_matches && skip_num > 0) + { + skip_num--; + continue; + } + + + best_mlen = (best_mlen > MINMATCH) ? best_mlen : MINMATCH; + + if (ctx->params.strategy == LZ5HC_optimal_price) + { + LZ5HC_Insert(ctx, inr); + match_num = LZ5HC_GetAllMatches(ctx, inr, ip, matchlimit, best_mlen, matches); + LZ5_LOG_PARSER("%d: LZ5HC_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + else + { + if (ctx->params.fullSearch < 2) + LZ5HC_BinTree_Insert(ctx, inr); + else + LZ5HC_BinTree_InsertFull(ctx, inr, matchlimit); + match_num = LZ5HC_BinTree_GetAllMatches(ctx, inr, matchlimit, best_mlen, matches); + LZ5_LOG_PARSER("%d: LZ5HC_BinTree_GetAllMatches match_num=%d\n", (int)(inr-source), match_num); + } + + + if (match_num > 0 && (size_t)matches[match_num-1].len > sufficient_len) + { + cur -= matches[match_num-1].back; + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto encode; + } + + // set prices using matches at position = cur + for (i = 0; i < match_num; i++) + { + mlen = (i>0) ? (size_t)matches[i-1].len+1 : best_mlen; + cur2 = cur - matches[i].back; + best_mlen = (cur2 + matches[i].len < LZ5_OPT_NUM) ? (size_t)matches[i].len : LZ5_OPT_NUM - cur2; + LZ5_LOG_PARSER("%d: Found1 cur=%d cur2=%d mlen=%d off=%d best_mlen=%d last_pos=%d\n", (int)(inr-source), cur, cur2, matches[i].len, matches[i].off, best_mlen, last_pos); + + if (mlen < (size_t)matches[i].back + 1) + mlen = matches[i].back + 1; + + while (mlen <= best_mlen) + { + if (opt[cur2].mlen == 1) + { + litlen = opt[cur2].litlen; + + if (cur2 != litlen) + price = opt[cur2 - litlen].price + LZ5HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + else + price = LZ5HC_get_price(llen + litlen, matches[i].off, mlen - MINMATCH) - llen; + } + else + { + litlen = 0; + price = opt[cur2].price + LZ5HC_get_price(litlen, matches[i].off, mlen - MINMATCH); + } + + LZ5_LOG_PARSER("%d: Found2 pred=%d mlen=%d best_mlen=%d off=%d price=%d litlen=%d price[%d]=%d\n", (int)(inr-source), matches[i].back, mlen, best_mlen, matches[i].off, price, litlen, cur - litlen, opt[cur - litlen].price); + // if (cur2 + mlen > last_pos || ((matches[i].off != opt[cur2 + mlen].off) && (price < opt[cur2 + mlen].price))) + if (cur2 + mlen > last_pos || price < (size_t)opt[cur2 + mlen].price) + { + SET_PRICE(cur2 + mlen, mlen, matches[i].off, litlen, price); + } + + mlen++; + } + } + } // for (skip_num = 0, cur = 1; cur <= last_pos; cur++) + + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + +encode: // cur, last_pos, best_mlen, best_off have to be set + for (i = 1; i <= last_pos; i++) + { + LZ5_LOG_PARSER("%d: price[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); + } + + LZ5_LOG_PARSER("%d: cur=%d/%d best_mlen=%d best_off=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, best_mlen, best_off, opt[cur].rep); + + opt[0].mlen = 1; + + while (1) + { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = (int)best_mlen; + opt[cur].off = (int)best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (i = 0; i <= last_pos;) + { + LZ5_LOG_PARSER("%d: price2[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+i), i, last_pos, opt[i].price, opt[i].off, opt[i].mlen, opt[i].litlen, opt[i].rep); + i += opt[i].mlen; + } + + cur = 0; + + while (cur < last_pos) + { + LZ5_LOG_PARSER("%d: price3[%d/%d]=%d off=%d mlen=%d litlen=%d rep=%d\n", (int)(ip-source+cur), cur, last_pos, opt[cur].price, opt[cur].off, opt[cur].mlen, opt[cur].litlen, opt[cur].rep); + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + + LZ5_LOG_ENCODE("%d: ENCODE literals=%d off=%d mlen=%d ", (int)(ip-source), (int)(ip-anchor), (int)(offset), mlen); + res = LZ5HC_encodeSequence(ctx, &ip, &op, &anchor, (int)mlen, ip - offset, limit, oend); + LZ5_LOG_ENCODE("out=%d\n", (int)((char*)op - dest)); + + if (res) return 0; + + LZ5_LOG_PARSER("%d: offset=%d rep=%d\n", (int)(ip-source), offset, ctx->last_off); + } + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + // if (inputSize > LASTLITERALS && lastRun < LASTLITERALS) { printf("ERROR: lastRun=%d\n", lastRun); } + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<base + ctx->dictLimit; + + /* init */ + ctx->inputBuffer = (const BYTE*)source; + ctx->outputBuffer = (const BYTE*)dest; + ctx->end += inputSize; + + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + LZ5HC_Insert(ctx, ip); + ml = LZ5HC_FindBestMatch (ctx, ip, matchlimit, (&ref)); + if (!ml) { ip++; continue; } + + { + int back = 0; + while ((ip + back > anchor) && (ref + back > lowPrefixPtr) && (ip[back - 1] == ref[back - 1])) back--; + ml -= back; + ip += back; + ref += back; + } + + /* saved, in case we would skip too much */ + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search: + if (ip+ml >= mflimit) goto _Encode; + + LZ5HC_Insert(ctx, ip); + ml2 = (int)LZ5HC_GetWiderMatch(ctx, ip + ml - 2, anchor, matchlimit, 0, &ref2, &start2); + if (ml2 == 0) goto _Encode; + + { + int price, best_price; + U32 off0=0, off1=0; + const uint8_t *pos, *best_pos; + + // find the lowest price for encoding ml bytes + best_pos = ip; + best_price = 1<<30; + off0 = (U32)(ip - ref); + off1 = (U32)(start2 - ref2); + + for (pos = ip + ml; pos >= start2; pos--) + { + int common0 = (int)(pos - ip); + if (common0 >= MINMATCH) + { + price = (int)LZ5_CODEWORD_COST(ip - anchor, (off0 == ctx->last_off) ? 0 : off0, common0 - MINMATCH); + + { + int common1 = (int)(start2 + ml2 - pos); + if (common1 >= MINMATCH) + price += (int)LZ5_CODEWORD_COST(0, (off1 == off0) ? 0 : (off1), common1 - MINMATCH); + else + price += LZ5_LIT_ONLY_COST(common1) - 1; + } + + if (price < best_price) + { + best_price = price; + best_pos = pos; + } + } + else + { + price = (int)LZ5_CODEWORD_COST(start2 - anchor, (off1 == ctx->last_off) ? 0 : off1, ml2 - MINMATCH); + + if (price < best_price) + best_pos = pos; + break; + } + } + // LZ5HC_DEBUG("%u: TRY last_off=%d literals=%u off=%u mlen=%u literals2=%u off2=%u mlen2=%u best=%d\n", (U32)(ip - ctx->inputBuffer), ctx->last_off, (U32)(ip - anchor), off0, (U32)ml, (U32)(start2 - anchor), off1, ml2, (U32)(best_pos - ip)); + ml = (int)(best_pos - ip); + } + + + if (ml < MINMATCH) + { + ip = start2; + ref = ref2; + ml = ml2; + goto _Search; + } + +_Encode: + + if (start0 < ip) + { + if (LZ5HC_more_profitable((ip - ref), ml,(start0 - ref0), ml0, (ref0 - ref), ctx->last_off)) + { + ip = start0; + ref = ref0; + ml = ml0; + } + } + + if (LZ5HC_encodeSequence(ctx, &ip, &op, &anchor, ml, ref, limit, oend)) return 0; + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<base + ctx->dictLimit; + U32* HashTable = ctx->hashTable; +#if MINMATCH == 3 + U32* HashTable3 = ctx->hashTable3; +#endif + const BYTE* const base = ctx->base; + U32* HashPos, *HashPos3; + + /* init */ + ctx->inputBuffer = (const BYTE*)source; + ctx->outputBuffer = (const BYTE*)dest; + ctx->end += inputSize; + + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + HashPos = &HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; +#if MINMATCH == 3 + HashPos3 = &HashTable3[LZ5HC_hash3Ptr(ip, ctx->params.hashLog3)]; + ml = LZ5HC_FindMatchFast (ctx, *HashPos, *HashPos3, ip, matchlimit, (&ref)); + *HashPos3 = (U32)(ip - base); +#else + ml = LZ5HC_FindMatchFast (ctx, *HashPos, 0, ip, matchlimit, (&ref)); +#endif + *HashPos = (U32)(ip - base); + + if (!ml) { ip++; continue; } + + if ((U32)(ip - ref) == ctx->last_off) { ml2=0; goto _Encode; } + + { + int back = 0; + while ((ip+back>anchor) && (ref+back > lowPrefixPtr) && (ip[back-1] == ref[back-1])) back--; + ml -= back; + ip += back; + ref += back; + } + +_Search: + if (ip+ml >= mflimit) goto _Encode; + + start2 = ip + ml - 2; + HashPos = &HashTable[LZ5HC_hashPtr(start2, ctx->params.hashLog, ctx->params.searchLength)]; + ml2 = LZ5HC_FindMatchFaster(ctx, *HashPos, start2, matchlimit, (&ref2)); + *HashPos = (U32)(start2 - base); + if (!ml2) goto _Encode; + + { + int back = 0; + while ((start2+back>ip) && (ref2+back > lowPrefixPtr) && (start2[back-1] == ref2[back-1])) back--; + ml2 -= back; + start2 += back; + ref2 += back; + } + + // LZ5HC_DEBUG("%u: TRY last_off=%d literals=%u off=%u mlen=%u literals2=%u off2=%u mlen2=%u best=%d\n", (U32)(ip - ctx->inputBuffer), ctx->last_off, (U32)(ip - anchor), off0, (U32)ml, (U32)(start2 - anchor), off1, ml2, (U32)(best_pos - ip)); + + if (ml2 <= ml) { ml2 = 0; goto _Encode; } + + if (start2 <= ip) + { + ip = start2; ref = ref2; ml = ml2; + ml2 = 0; + goto _Encode; + } + + if (start2 - ip < 3) + { + ip = start2; ref = ref2; ml = ml2; + ml2 = 0; + goto _Search; + } + + + if (start2 < ip + ml) + { + int correction = ml - (int)(start2 - ip); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < 3) { ml2 = 0; } + } + +_Encode: + if (LZ5HC_encodeSequence(ctx, &ip, &op, &anchor, ml, ref, limit, oend)) return 0; + + if (ml2) + { + ip = start2; ref = ref2; ml = ml2; + ml2 = 0; + goto _Search; + } + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<base + ctx->dictLimit; + const BYTE* const base = ctx->base; + U32* HashPos; + U32* HashTable = ctx->hashTable; + const int accel = (ctx->params.searchNum>0)?ctx->params.searchNum:1; + + /* init */ + ctx->inputBuffer = (const BYTE*)source; + ctx->outputBuffer = (const BYTE*)dest; + ctx->end += inputSize; + + ip++; + + /* Main Loop */ + while (ip < mflimit) + { + HashPos = &HashTable[LZ5HC_hashPtr(ip, ctx->params.hashLog, ctx->params.searchLength)]; + ml = LZ5HC_FindMatchFastest (ctx, *HashPos, ip, matchlimit, (&ref)); + *HashPos = (U32)(ip - base); + if (!ml) { ip+=accel; continue; } + + { + int back = 0; + while ((ip + back > anchor) && (ref + back > lowPrefixPtr) && (ip[back - 1] == ref[back - 1])) back--; + ml -= back; + ip += back; + ref += back; + } + + if (LZ5HC_encodeSequence(ctx, &ip, &op, &anchor, ml, ref, limit, oend)) return 0; + + } + + /* Encode Last Literals */ + { + int lastRun = (int)(iend - anchor); + if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */ + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (BYTE)(lastRun<params.strategy) + { + default: + case LZ5HC_fast: + return LZ5HC_compress_fast(ctx, source, dest, inputSize, maxOutputSize, limit); + case LZ5HC_price_fast: + return LZ5HC_compress_price_fast(ctx, source, dest, inputSize, maxOutputSize, limit); + case LZ5HC_lowest_price: + return LZ5HC_compress_lowest_price(ctx, source, dest, inputSize, maxOutputSize, limit); + case LZ5HC_optimal_price: + case LZ5HC_optimal_price_bt: + return LZ5HC_compress_optimal_price(ctx, (const BYTE* )source, dest, inputSize, maxOutputSize, limit); + } +} + + +int LZ5_sizeofStateHC(void) { return sizeof(LZ5HC_Data_Structure); } + +int LZ5_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) +{ + if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */ + LZ5HC_init ((LZ5HC_Data_Structure*)state, (const BYTE*)src); + if (maxDstSize < LZ5_compressBound(srcSize)) + return LZ5HC_compress_generic (state, src, dst, srcSize, maxDstSize, limitedOutput); + else + return LZ5HC_compress_generic (state, src, dst, srcSize, maxDstSize, noLimit); +} + + +int LZ5_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel) +{ + LZ5HC_Data_Structure state; + LZ5HC_Data_Structure* const statePtr = &state; + int cSize = 0; + + if (!LZ5_alloc_mem_HC(statePtr, compressionLevel)) + return 0; + cSize = LZ5_compress_HC_extStateHC(statePtr, src, dst, srcSize, maxDstSize); + + LZ5_free_mem_HC(statePtr); + + return cSize; +} + + + +/************************************** +* Streaming Functions +**************************************/ +/* allocation */ +LZ5_streamHC_t* LZ5_createStreamHC(int compressionLevel) +{ + LZ5_streamHC_t* statePtr = (LZ5_streamHC_t*)malloc(sizeof(LZ5_streamHC_t)); + if (!statePtr) + return NULL; + + if (!LZ5_alloc_mem_HC((LZ5HC_Data_Structure*)statePtr, compressionLevel)) + { + FREEMEM(statePtr); + return NULL; + } + return statePtr; +} + +int LZ5_freeStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr) +{ + LZ5HC_Data_Structure* statePtr = (LZ5HC_Data_Structure*)LZ5_streamHCPtr; + if (statePtr) + { + LZ5_free_mem_HC(statePtr); + free(LZ5_streamHCPtr); + } + return 0; +} + + +/* initialization */ +void LZ5_resetStreamHC (LZ5_streamHC_t* LZ5_streamHCPtr) +{ + LZ5_STATIC_ASSERT(sizeof(LZ5HC_Data_Structure) <= sizeof(LZ5_streamHC_t)); /* if compilation fails here, LZ5_STREAMHCSIZE must be increased */ + ((LZ5HC_Data_Structure*)LZ5_streamHCPtr)->base = NULL; +} + +int LZ5_loadDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, const char* dictionary, int dictSize) +{ + LZ5HC_Data_Structure* ctxPtr = (LZ5HC_Data_Structure*) LZ5_streamHCPtr; + if (dictSize > LZ5_DICT_SIZE) + { + dictionary += dictSize - LZ5_DICT_SIZE; + dictSize = LZ5_DICT_SIZE; + } + LZ5HC_init (ctxPtr, (const BYTE*)dictionary); + if (dictSize >= 4) LZ5HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3)); + ctxPtr->end = (const BYTE*)dictionary + dictSize; + return dictSize; +} + + +/* compression */ + +static void LZ5HC_setExternalDict(LZ5HC_Data_Structure* ctxPtr, const BYTE* newBlock) +{ + if (ctxPtr->end >= ctxPtr->base + 4) + LZ5HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */ + /* Only one memory segment for extDict, so any previous extDict is lost at this stage */ + ctxPtr->lowLimit = ctxPtr->dictLimit; + ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base); + ctxPtr->dictBase = ctxPtr->base; + ctxPtr->base = newBlock - ctxPtr->dictLimit; + ctxPtr->end = newBlock; + ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */ +} + +static int LZ5_compressHC_continue_generic (LZ5HC_Data_Structure* ctxPtr, + const char* source, char* dest, + int inputSize, int maxOutputSize, limitedOutput_directive limit) +{ + /* auto-init if forgotten */ + if (ctxPtr->base == NULL) + LZ5HC_init (ctxPtr, (const BYTE*) source); + + /* Check overflow */ + if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB) + { + size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit; + if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; + + LZ5_loadDictHC((LZ5_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize); + } + + /* Check if blocks follow each other */ + if ((const BYTE*)source != ctxPtr->end) + LZ5HC_setExternalDict(ctxPtr, (const BYTE*)source); + + /* Check overlapping input/dictionary space */ + { + const BYTE* sourceEnd = (const BYTE*) source + inputSize; + const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit; + const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit; + if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd)) + { + if (sourceEnd > dictEnd) sourceEnd = dictEnd; + ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase); + if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit; + } + } + + return LZ5HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, limit); +} + +int LZ5_compress_HC_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize) +{ + if (maxOutputSize < LZ5_compressBound(inputSize)) + return LZ5_compressHC_continue_generic ((LZ5HC_Data_Structure*)LZ5_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput); + else + return LZ5_compressHC_continue_generic ((LZ5HC_Data_Structure*)LZ5_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit); +} + + +/* dictionary saving */ + +int LZ5_saveDictHC (LZ5_streamHC_t* LZ5_streamHCPtr, char* safeBuffer, int dictSize) +{ + LZ5HC_Data_Structure* streamPtr = (LZ5HC_Data_Structure*)LZ5_streamHCPtr; + int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit)); + if (dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE; + // if (dictSize < 4) dictSize = 0; + if (dictSize > prefixSize) dictSize = prefixSize; + memmove(safeBuffer, streamPtr->end - dictSize, dictSize); + { + U32 endIndex = (U32)(streamPtr->end - streamPtr->base); + streamPtr->end = (const BYTE*)safeBuffer + dictSize; + streamPtr->base = streamPtr->end - endIndex; + streamPtr->dictLimit = endIndex - dictSize; + streamPtr->lowLimit = endIndex - dictSize; + if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit; + } + return dictSize; +} + +/*********************************** +* Deprecated Functions +***********************************/ +/* Deprecated compression functions */ +/* These functions are planned to start generate warnings by r132 approximately */ +int LZ5_compressHC(const char* src, char* dst, int srcSize) { return LZ5_compress_HC (src, dst, srcSize, LZ5_compressBound(srcSize), 0); } +int LZ5_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC(src, dst, srcSize, maxDstSize, 0); } +int LZ5_compressHC_continue (LZ5_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ5_compress_HC_continue (ctx, src, dst, srcSize, LZ5_compressBound(srcSize)); } +int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); } +int LZ5_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ5_compress_HC_extStateHC (state, src, dst, srcSize, LZ5_compressBound(srcSize)); } +int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ5_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize); } diff --git a/C/lz5/lz5hc.h b/C/lz5/lz5hc.h new file mode 100644 index 00000000..c7f4a00f --- /dev/null +++ b/C/lz5/lz5hc.h @@ -0,0 +1,178 @@ +/* + LZ5 HC - High Compression Mode of LZ5 + Header File + Copyright (C) 2011-2015, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ5 source repository : https://github.com/inikep/lz5 + - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c +*/ +#pragma once + + +#if defined (__cplusplus) +extern "C" { +#endif + +/***************************** +* Includes +*****************************/ +#include /* size_t */ + + +/************************************** +* Block Compression +**************************************/ +int LZ5_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel); +/* +LZ5_compress_HC : + Destination buffer 'dst' must be already allocated. + Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible) + Worst size evaluation is provided by function LZ5_compressBound() (see "lz5.h") + srcSize : Max supported value is LZ5_MAX_INPUT_SIZE (see "lz5.h") + compressionLevel : Recommended values are between 4 and 9, although any value between 0 and LZ5HC_MAX_CLEVEL (equal to 15) will work. + 0 means "use default value" (see lz5hc.c). + Values >LZ5HC_MAX_CLEVEL behave the same as LZ5HC_MAX_CLEVEL. + return : the number of bytes written into buffer 'dst' + or 0 if compression fails. +*/ + + +/* Note : + Decompression functions are provided within LZ5 source code (see "lz5.h") (BSD license) +*/ + +typedef struct LZ5HC_Data_s LZ5HC_Data_Structure; + +int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr, int compressionLevel); +void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr); + +int LZ5_sizeofStateHC(void); +int LZ5_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize); +/* +LZ5_compress_HC_extStateHC() : + Use this function if you prefer to manually allocate memory for compression tables. + To know how much memory must be allocated for the compression tables, use : + int LZ5_sizeofStateHC(); + + Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly). + + The allocated memory can then be provided to the compression functions using 'void* state' parameter. + LZ5_compress_HC_extStateHC() is equivalent to previously described function. + It just uses externally allocated memory for stateHC. +*/ + + +/************************************** +* Streaming Compression +**************************************/ +#define LZ5_STREAMHCSIZE 262192 +#define LZ5_STREAMHCSIZE_SIZET (LZ5_STREAMHCSIZE / sizeof(size_t)) +typedef struct { size_t table[LZ5_STREAMHCSIZE_SIZET]; } LZ5_streamHC_t; +/* + LZ5_streamHC_t + This structure allows static allocation of LZ5 HC streaming state. + State must then be initialized using LZ5_resetStreamHC() before first use. + + Static allocation should only be used in combination with static linking. + If you want to use LZ5 as a DLL, please use construction functions below, which are future-proof. +*/ + + +LZ5_streamHC_t* LZ5_createStreamHC(int compressionLevel); +int LZ5_freeStreamHC (LZ5_streamHC_t* streamHCPtr); +/* + These functions create and release memory for LZ5 HC streaming state. + Newly created states are already initialized. + Existing state space can be re-used anytime using LZ5_resetStreamHC(). + If you use LZ5 as a DLL, use these functions instead of static structure allocation, + to avoid size mismatch between different versions. +*/ + +void LZ5_resetStreamHC (LZ5_streamHC_t* streamHCPtr); +int LZ5_loadDictHC (LZ5_streamHC_t* streamHCPtr, const char* dictionary, int dictSize); + +int LZ5_compress_HC_continue (LZ5_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize); + +int LZ5_saveDictHC (LZ5_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize); + +/* + These functions compress data in successive blocks of any size, using previous blocks as dictionary. + One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks. + There is an exception for ring buffers, which can be smaller 64 KB. + Such case is automatically detected and correctly handled by LZ5_compress_HC_continue(). + + Before starting compression, state must be properly initialized, using LZ5_resetStreamHC(). + A first "fictional block" can then be designated as initial dictionary, using LZ5_loadDictHC() (Optional). + + Then, use LZ5_compress_HC_continue() to compress each successive block. + It works like LZ5_compress_HC(), but use previous memory blocks as dictionary to improve compression. + Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression. + As a reminder, size 'dst' buffer to handle worst cases, using LZ5_compressBound(), to ensure success of compression operation. + + If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block, + you must save it to a safer memory space, using LZ5_saveDictHC(). + Return value of LZ5_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'. +*/ + + + +/************************************** +* Deprecated Functions +**************************************/ +/* Deprecate Warnings */ +/* Should these warnings messages be a problem, + it is generally possible to disable them, + with -Wno-deprecated-declarations for gcc + or _CRT_SECURE_NO_WARNINGS in Visual for example. + You can also define LZ5_DEPRECATE_WARNING_DEFBLOCK. */ +#ifndef LZ5_DEPRECATE_WARNING_DEFBLOCK +# define LZ5_DEPRECATE_WARNING_DEFBLOCK +# if (LZ5_GCC_VERSION >= 405) || defined(__clang__) +# define LZ5_DEPRECATED(message) __attribute__((deprecated(message))) +# elif (LZ5_GCC_VERSION >= 301) +# define LZ5_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define LZ5_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement LZ5_DEPRECATED for this compiler") +# define LZ5_DEPRECATED(message) +# endif +#endif // LZ5_DEPRECATE_WARNING_DEFBLOCK + +/* compression functions */ +/* these functions are planned to trigger warning messages by r132 approximately */ +int LZ5_compressHC (const char* source, char* dest, int inputSize); +int LZ5_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ5_compressHC_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize); +int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize); +int LZ5_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize); +int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); + +#if defined (__cplusplus) +} +#endif diff --git a/C/lz5/lz5xxhash.c b/C/lz5/lz5xxhash.c new file mode 100644 index 00000000..e6fb8f14 --- /dev/null +++ b/C/lz5/lz5xxhash.c @@ -0,0 +1,915 @@ +/* +xxHash - Fast Hash algorithm +Copyright (C) 2012-2015, Yann Collet + +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/************************************** +* Tuning parameters +**************************************/ +/* Unaligned memory access is automatically enabled for "common" CPU, such as x86. + * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. + * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. + * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). + */ +#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_USE_UNALIGNED_ACCESS 1 +#endif + +/* XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/* XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1. + * It will improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#define XXH_FORCE_NATIVE_FORMAT 0 + + +/************************************** +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/************************************** +* Includes & Memory related functions +***************************************/ +#include "xxhash.h" +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + + +/************************************** +* Basic Types +***************************************/ +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; +#endif + +static U32 XXH_read32(const void* memPtr) +{ + U32 val32; + memcpy(&val32, memPtr, 4); + return val32; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val64; + memcpy(&val64, memPtr, 8); + return val64; +} + + + +/****************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/*************************************** +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; +#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */ +static const int one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one)) +#endif + + +/***************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + + +/*************************************** +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */ + + +/*************************************** +* Constants +***************************************/ +#define PRIME32_1 2654435761U +#define PRIME32_2 2246822519U +#define PRIME32_3 3266489917U +#define PRIME32_4 668265263U +#define PRIME32_5 374761393U + +#define PRIME64_1 11400714785074694791ULL +#define PRIME64_2 14029467366897019727ULL +#define PRIME64_3 1609587929392839161ULL +#define PRIME64_4 9650029242287828579ULL +#define PRIME64_5 2870177450012600261ULL + + +/***************************** +* Simple Hash Functions +*****************************/ +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do + { + v1 += XXH_get32bits(p) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_get32bits(p) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_get32bits(p) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_get32bits(p) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } + else + { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) + { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +unsigned XXH32 (const void* input, size_t len, unsigned seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, input, len); + return XXH32_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */ + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) + { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do + { + v1 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + v2 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + v3 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + v4 += XXH_get64bits(p) * PRIME64_2; + p+=8; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + } + while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64 * PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64 * PRIME64_1 + PRIME64_4; + } + else + { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_get64bits(p); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, input, len); + return XXH64_digest(&state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + +# if !defined(XXH_USE_UNALIGNED_ACCESS) + if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */ + { + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } +# endif + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + +/**************************************************** +* Advanced Hash Functions +****************************************************/ + +/*** Allocation ***/ +typedef struct +{ + U64 total_len; + U32 seed; + U32 v1; + U32 v2; + U32 v3; + U32 v4; + U32 mem32[4]; /* defined as U32 for alignment */ + U32 memsize; +} XXH_istate32_t; + +typedef struct +{ + U64 total_len; + U64 seed; + U64 v1; + U64 v2; + U64 v3; + U64 v4; + U64 mem64[4]; /* defined as U64 for alignment */ + U32 memsize; +} XXH_istate64_t; + + +XXH32_state_t* XXH32_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH64_state_t* XXH64_createState(void) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed) +{ + XXH_istate32_t* state = (XXH_istate32_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME32_1 + PRIME32_2; + state->v2 = seed + PRIME32_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME32_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + +XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed) +{ + XXH_istate64_t* state = (XXH_istate64_t*) state_in; + state->seed = seed; + state->v1 = seed + PRIME64_1 + PRIME64_2; + state->v2 = seed + PRIME64_2; + state->v3 = seed + 0; + state->v4 = seed - PRIME64_1; + state->total_len = 0; + state->memsize = 0; + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate32_t* state = (XXH_istate32_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 16) /* fill in tmp buffer */ + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) /* some data left from previous update */ + { + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { + const U32* p32 = state->mem32; + state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v1 = XXH_rotl32(state->v1, 13); + state->v1 *= PRIME32_1; + p32++; + state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v2 = XXH_rotl32(state->v2, 13); + state->v2 *= PRIME32_1; + p32++; + state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v3 = XXH_rotl32(state->v3, 13); + state->v3 *= PRIME32_1; + p32++; + state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; + state->v4 = XXH_rotl32(state->v4, 13); + state->v4 *= PRIME32_1; + p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) + { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do + { + v1 += XXH_readLE32(p, endian) * PRIME32_2; + v1 = XXH_rotl32(v1, 13); + v1 *= PRIME32_1; + p+=4; + v2 += XXH_readLE32(p, endian) * PRIME32_2; + v2 = XXH_rotl32(v2, 13); + v2 *= PRIME32_1; + p+=4; + v3 += XXH_readLE32(p, endian) * PRIME32_2; + v3 = XXH_rotl32(v3, 13); + v3 *= PRIME32_1; + p+=4; + v4 += XXH_readLE32(p, endian) * PRIME32_2; + v4 = XXH_rotl32(v4, 13); + v4 *= PRIME32_1; + p+=4; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem32, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian) +{ + const XXH_istate32_t* state = (const XXH_istate32_t*) state_in; + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->total_len >= 16) + { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } + else + { + h32 = state->seed + PRIME32_5; + } + + h32 += (U32) state->total_len; + + while (p+4<=bEnd) + { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +U32 XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian) +{ + XXH_istate64_t * state = (XXH_istate64_t *) state_in; + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) /* fill in tmp buffer */ + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) /* some data left from previous update */ + { + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + { + const U64* p64 = state->mem64; + state->v1 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v1 = XXH_rotl64(state->v1, 31); + state->v1 *= PRIME64_1; + p64++; + state->v2 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v2 = XXH_rotl64(state->v2, 31); + state->v2 *= PRIME64_1; + p64++; + state->v3 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v3 = XXH_rotl64(state->v3, 31); + state->v3 *= PRIME64_1; + p64++; + state->v4 += XXH_readLE64(p64, endian) * PRIME64_2; + state->v4 = XXH_rotl64(state->v4, 31); + state->v4 *= PRIME64_1; + p64++; + } + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) + { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do + { + v1 += XXH_readLE64(p, endian) * PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + p+=8; + v2 += XXH_readLE64(p, endian) * PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + p+=8; + v3 += XXH_readLE64(p, endian) * PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + p+=8; + v4 += XXH_readLE64(p, endian) * PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + p+=8; + } + while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) + { + XXH_memcpy(state->mem64, p, bEnd-p); + state->memsize = (int)(bEnd-p); + } + + return XXH_OK; +} + +XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian) +{ + const XXH_istate64_t * state = (const XXH_istate64_t *) state_in; + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) + { + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + + v1 *= PRIME64_2; + v1 = XXH_rotl64(v1, 31); + v1 *= PRIME64_1; + h64 ^= v1; + h64 = h64*PRIME64_1 + PRIME64_4; + + v2 *= PRIME64_2; + v2 = XXH_rotl64(v2, 31); + v2 *= PRIME64_1; + h64 ^= v2; + h64 = h64*PRIME64_1 + PRIME64_4; + + v3 *= PRIME64_2; + v3 = XXH_rotl64(v3, 31); + v3 *= PRIME64_1; + h64 ^= v3; + h64 = h64*PRIME64_1 + PRIME64_4; + + v4 *= PRIME64_2; + v4 = XXH_rotl64(v4, 31); + v4 *= PRIME64_1; + h64 ^= v4; + h64 = h64*PRIME64_1 + PRIME64_4; + } + else + { + h64 = state->seed + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) + { + U64 k1 = XXH_readLE64(p, endian); + k1 *= PRIME64_2; + k1 = XXH_rotl64(k1,31); + k1 *= PRIME64_1; + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) + { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + diff --git a/C/lz5/mem.h b/C/lz5/mem.h new file mode 100644 index 00000000..ff1221e4 --- /dev/null +++ b/C/lz5/mem.h @@ -0,0 +1,452 @@ +/* ****************************************************************** + mem.h + low-level memory access routines + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/****************************************** +* Includes +******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + + + +/****************************************** +* Compiler-specific +******************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#elif defined(__GNUC__) +# define MEM_STATIC static __attribute__((unused)) +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + + +/**************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets generating assembly depending on alignment. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif // MEM_FORCE_MEMORY_ACCESS + + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) + { + MEM_write16(memPtr, val); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + if (MEM_isLittleEndian()) + { + U32 val32 = 0; + memcpy(&val32, memPtr, 3); + return val32; + } + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U32)(p[0] + (p[1]<<8) + (p[2]<<16)); + } +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 value) +{ + if (MEM_isLittleEndian()) + { + memcpy(memPtr, &value, 3); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE) value; + p[1] = (BYTE)(value>>8); + p[2] = (BYTE)(value>>16); + } +} + + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24)); + } +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + { + MEM_write32(memPtr, val32); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val32; + p[1] = (BYTE)(val32>>8); + p[2] = (BYTE)(val32>>16); + p[3] = (BYTE)(val32>>24); + } +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + { + const BYTE* p = (const BYTE*)memPtr; + return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24) + + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56)); + } +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + { + MEM_write64(memPtr, val64); + } + else + { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val64; + p[1] = (BYTE)(val64>>8); + p[2] = (BYTE)(val64>>16); + p[3] = (BYTE)(val64>>24); + p[4] = (BYTE)(val64>>32); + p[5] = (BYTE)(val64>>40); + p[6] = (BYTE)(val64>>48); + p[7] = (BYTE)(val64>>56); + } +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + + +#if MINMATCH == 3 + #define MEM_read24(ptr) (U32)(MEM_read32(ptr)<<8) +#else + #define MEM_read24(ptr) (U32)(MEM_read32(ptr)) +#endif + + +/* ************************************** +* Function body to include for inlining +****************************************/ +static size_t MEM_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; } + +#define MIN(a,b) ((a)<(b) ? (a) : (b)) + + +/*static unsigned MEM_highbit(U32 val) +{ +# if defined(_MSC_VER) // Visual + unsigned long r=0; + _BitScanReverse(&r, val); + return (unsigned)r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) // GCC Intrinsic + return 31 - __builtin_clz(val); +# else // Software version + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +}*/ + + +MEM_STATIC unsigned MEM_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) + { + if (MEM_64bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (int)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } + else /* Big Endian CPU */ + { + if (MEM_32bits()) + { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } + else /* 32 bits */ + { +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } + } +} + + +MEM_STATIC size_t MEM_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit) +{ + const BYTE* const pStart = pIn; + + while ((pIn /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/***************************** +* Namespace Emulation +*****************************/ +/* Motivations : + +If you need to include xxHash into your library, +but wish to avoid xxHash symbols to be present on your library interface +in an effort to avoid potential name collision if another library also includes xxHash, + +you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash +with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values). + +Note that no change is required within the calling program : +it can still call xxHash functions using their regular name. +They will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +#endif + + +/***************************** +* Simple Hash Functions +*****************************/ + +unsigned int XXH32 (const void* input, size_t length, unsigned seed); +unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed); + +/* +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + This function successfully passes all SMHasher tests. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + Faster on 64-bits systems. Slower on 32-bits systems. +*/ + + + +/***************************** +* Advanced Hash Functions +*****************************/ +typedef struct { long long ll[ 6]; } XXH32_state_t; +typedef struct { long long ll[11]; } XXH64_state_t; + +/* +These structures allow static allocation of XXH states. +States must then be initialized using XXHnn_reset() before first use. + +If you prefer dynamic allocation, please refer to functions below. +*/ + +XXH32_state_t* XXH32_createState(void); +XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH64_state_t* XXH64_createState(void); +XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/* +These functions create and release memory for XXH state. +States must then be initialized using XXHnn_reset() before first use. +*/ + + +XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed); +XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +unsigned int XXH32_digest (const XXH32_state_t* statePtr); + +XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +unsigned long long XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions calculate the xxHash of an input provided in multiple smaller packets, +as opposed to an input provided as a single block. + +XXH state space must first be allocated, using either static or dynamic method provided above. + +Start a new hash by initializing state with a seed, using XXHnn_reset(). + +Then, feed the hash state by calling XXHnn_update() as many times as necessary. +Obviously, input must be valid, meaning allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, you can produce a hash anytime, by using XXHnn_digest(). +This function returns the final nn-bits hash. +You can nonetheless continue feeding the hash state with more input, +and therefore get some new hashes, by calling again XXHnn_digest(). + +When you are done, don't forget to free XXH state space, using typically XXHnn_freeState(). +*/ + + +#if defined (__cplusplus) +} +#endif diff --git a/C/zstd/error_private.c b/C/zstd/error_private.c new file mode 100644 index 00000000..a0fa1724 --- /dev/null +++ b/C/zstd/error_private.c @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} diff --git a/C/zstd/error_private.h b/C/zstd/error_private.h index d27e15af..1bc2e495 100644 --- a/C/zstd/error_private.h +++ b/C/zstd/error_private.h @@ -21,7 +21,7 @@ extern "C" { * Dependencies ******************************************/ #include /* size_t */ -#include "error_public.h" /* enum list */ +#include "zstd_errors.h" /* enum list */ /* **************************************** @@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu * Error Strings ******************************************/ -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(version_unsupported): return "Version not supported"; - case PREFIX(parameter_unknown): return "Unknown parameter type"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ ERR_STATIC const char* ERR_getErrorName(size_t code) { diff --git a/C/zstd/fse.h b/C/zstd/fse.h index 720d54b1..cecb1aef 100644 --- a/C/zstd/fse.h +++ b/C/zstd/fse.h @@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt BIT_flushBits(bitC); } + /* ====== Decompression ====== */ typedef struct { @@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ -#define FSE_MAX_MEMORY_USAGE 14 -#define FSE_DEFAULT_MEMORY_USAGE 13 +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif /*!FSE_MAX_SYMBOL_VALUE : * Maximum symbol value authorized. * Required for proper stack allocation */ -#define FSE_MAX_SYMBOL_VALUE 255 - +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif /* ************************************************************** * template functions type & suffix diff --git a/C/zstd/zstd.h b/C/zstd/zstd.h index dd3f5df4..c2f5f5d4 100644 --- a/C/zstd/zstd.h +++ b/C/zstd/zstd.h @@ -33,7 +33,7 @@ extern "C" { /*======= Version =======*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 1 -#define ZSTD_VERSION_RELEASE 0 +#define ZSTD_VERSION_RELEASE 1 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str diff --git a/C/zstd/zstd_compress.c b/C/zstd/zstd_compress.c index 94f4b5a2..e71873b0 100644 --- a/C/zstd/zstd_compress.c +++ b/C/zstd/zstd_compress.c @@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co const U32 dictLimit = zc->dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; - const BYTE* match = base + matchIndex; + const BYTE* match; const U32 current = (U32)(ip-base); const U32 btLow = btMask >= current ? 0 : current - btMask; U32* smallerPtr = bt + 2*(current&btMask); @@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, BYTE* op = ostart; U32 const maxDist = 1 << cctx->params.cParams.windowLog; - if (cctx->params.fParams.checksumFlag) + if (cctx->params.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); while (remaining) { @@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); diff --git a/C/zstd/zstd_decompress.c b/C/zstd/zstd_decompress.c index 47b5f42c..990a6491 100644 --- a/C/zstd/zstd_decompress.c +++ b/C/zstd/zstd_decompress.c @@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t if (!singleSegment) { BYTE const wlByte = ip[pos++]; U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; - if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported); + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ windowSize = (1U << windowLog); windowSize += (windowSize >> 3) * (wlByte&7); } @@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t case 3 : frameContentSize = MEM_readLE64(ip+pos); break; } if (!windowSize) windowSize = (U32)frameContentSize; - if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported); + if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); fparamsPtr->frameContentSize = frameContentSize; fparamsPtr->windowSize = windowSize; fparamsPtr->dictID = dictID; @@ -878,7 +878,12 @@ size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { @@ -1397,7 +1402,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu return NULL; } - memcpy(dictContent, dict, dictSize); + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); if (ZSTD_isError(errorCode)) { ZSTD_free(dictContent, customMem); @@ -1568,7 +1575,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, switch(paramType) { default : return ERROR(parameter_unknown); - case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break; + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; } return 0; } @@ -1649,7 +1656,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); diff --git a/C/zstd/zstd_errors.h b/C/zstd/zstd_errors.h new file mode 100644 index 00000000..50dc4f72 --- /dev/null +++ b/C/zstd/zstd_errors.h @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/*-**************************************** +* error codes list +******************************************/ +typedef enum { + ZSTD_error_no_error, + ZSTD_error_GENERIC, + ZSTD_error_prefix_unknown, + ZSTD_error_version_unsupported, + ZSTD_error_parameter_unknown, + ZSTD_error_frameParameter_unsupported, + ZSTD_error_frameParameter_unsupportedBy32bits, + ZSTD_error_frameParameter_windowTooLarge, + ZSTD_error_compressionParameter_unsupported, + ZSTD_error_init_missing, + ZSTD_error_memory_allocation, + ZSTD_error_stage_wrong, + ZSTD_error_dstSize_tooSmall, + ZSTD_error_srcSize_wrong, + ZSTD_error_corruption_detected, + ZSTD_error_checksum_wrong, + ZSTD_error_tableLog_tooLarge, + ZSTD_error_maxSymbolValue_tooLarge, + ZSTD_error_maxSymbolValue_tooSmall, + ZSTD_error_dictionary_corrupted, + ZSTD_error_dictionary_wrong, + ZSTD_error_maxCode +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +const char* ZSTD_getErrorString(ZSTD_ErrorCode code); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/C/zstd/zstd_opt.h b/C/zstd/zstd_opt.h index cb587290..cea67056 100644 --- a/C/zstd/zstd_opt.h +++ b/C/zstd/zstd_opt.h @@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); { U32 i; for (i=0; irep[i]; } - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, ctx->nextToUpdate3 = ctx->nextToUpdate; ZSTD_rescaleFreqs(seqStorePtr); ip += (ip==prefixStart); - inr = ip; + //inr = ip; /* Match Loop */ while (ip < ilimit) { @@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, U32 current = (U32)(ip-base); memset(opt, 0, sizeof(ZSTD_optimal_t)); last_pos = 0; - inr = ip; + //inr = ip; opt[0].litlen = (U32)(ip - anchor); /* check repCode */ diff --git a/C/zstd/zstd_v04.c b/C/zstd/zstd_v04.c index c9dcb94e..05e40aac 100644 --- a/C/zstd/zstd_v04.c +++ b/C/zstd/zstd_v04.c @@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) diff --git a/C/zstd/zstd_v05.c b/C/zstd/zstd_v05.c index 5027e2b8..96ffceb9 100644 --- a/C/zstd/zstd_v05.c +++ b/C/zstd/zstd_v05.c @@ -3312,7 +3312,12 @@ static size_t ZSTDv05_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/C/zstd/zstd_v06.c b/C/zstd/zstd_v06.c index d9e89f80..96a84d3e 100644 --- a/C/zstd/zstd_v06.c +++ b/C/zstd/zstd_v06.c @@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_8) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_8 */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/C/zstd/zstd_v07.c b/C/zstd/zstd_v07.c index f4c8073f..62285238 100644 --- a/C/zstd/zstd_v07.c +++ b/C/zstd/zstd_v07.c @@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op, op = oLitEnd + length1; sequence.matchLength -= length1; match = base; + if (op > oend_w) { + memmove(op, match, sequence.matchLength); + return sequenceLength; + } } } + /* Requirement: op <= oend_w */ /* match within prefix */ if (sequence.offset < 8) { diff --git a/C/zstdmt/README.md b/C/zstdmt/README.md new file mode 100644 index 00000000..32ad7cdb --- /dev/null +++ b/C/zstdmt/README.md @@ -0,0 +1,66 @@ + +# Multithreading Library for [LZ4], [LZ5] and [ZStandard] + + +### Compression +``` +typedef struct { + void *buf; /* ptr to data */ + size_t size; /* current filled in buf */ + size_t allocated; /* length of buf */ +} LZ4MT_Buffer; + +/** + * reading and writing functions + * - you can use stdio functions or plain read/write ... + * - a sample is given in 7-Zip ZS or lz4mt.c + */ +typedef int (fn_read) (void *args, LZ4MT_Buffer * in); +typedef int (fn_write) (void *args, LZ4MT_Buffer * out); + +typedef struct { + fn_read *fn_read; + void *arg_read; + fn_write *fn_write; + void *arg_write; +} LZ4MT_RdWr_t; + +typedef struct LZ4MT_CCtx_s LZ4MT_CCtx; + +/* 1) allocate new cctx */ +LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize); + +/* 2) threaded compression */ +size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/* 3) get some statistic */ +size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx); + +/* 4) free cctx */ +void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx); +``` + +### Decompression +``` +typedef struct LZ4MT_DCtx_s LZ4MT_DCtx; + +/* 1) allocate new cctx */ +LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize); + +/* 2) threaded compression */ +size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/* 3) get some statistic */ +size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx); + +/* 4) free cctx */ +void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx); +``` + +## Todo + +- add Makefile diff --git a/C/zstdmt/list.h b/C/zstdmt/list.h new file mode 100644 index 00000000..7ecbce99 --- /dev/null +++ b/C/zstdmt/list.h @@ -0,0 +1,130 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * - removed unneeded stuff + * - added win32 compatibility + * + * Copyright (c) 2013 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Taylor R. Campbell. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef LIST_H +#define LIST_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include "mem.h" + +struct list_head { + struct list_head *prev; + struct list_head *next; +}; + +MEM_STATIC void INIT_LIST_HEAD(struct list_head *head) +{ + head->prev = head; + head->next = head; +} + +MEM_STATIC struct list_head *list_first(const struct list_head *head) +{ + return head->next; +} + +MEM_STATIC struct list_head *list_last(const struct list_head *head) +{ + return head->prev; +} + +MEM_STATIC struct list_head *list_next(const struct list_head *node) +{ + return node->next; +} + +MEM_STATIC struct list_head *list_prev(const struct list_head *node) +{ + return node->prev; +} + +MEM_STATIC void __list_add_between(struct list_head *prev, + struct list_head *node, struct list_head *next) +{ + prev->next = node; + node->prev = prev; + node->next = next; + next->prev = node; +} + +MEM_STATIC void list_add(struct list_head *node, struct list_head *head) +{ + __list_add_between(head, node, head->next); +} + +MEM_STATIC void list_add_tail(struct list_head *node, struct list_head *head) +{ + __list_add_between(head->prev, node, head); +} + +MEM_STATIC void list_del(struct list_head *entry) +{ + entry->prev->next = entry->next; + entry->next->prev = entry->prev; +} + +MEM_STATIC int list_empty(const struct list_head *head) +{ + return (head->next == head); +} + +MEM_STATIC void list_move(struct list_head *node, struct list_head *head) +{ + list_del(node); + list_add(node, head); +} + +MEM_STATIC void list_move_tail(struct list_head *node, struct list_head *head) +{ + list_del(node); + list_add_tail(node, head); +} + +#ifndef CONTAINING_RECORD +#define CONTAINING_RECORD(ptr, type, field) \ + ((type*)((char*)(ptr) - (char*)(&((type*)0)->field))) +#endif + +#define list_entry(ptr, type, member) CONTAINING_RECORD(ptr, type, member) +#define list_for_each(var, head) \ + for ((var) = list_first((head)); (var) != (head); (var) = list_next((var))) + +#if defined (__cplusplus) +} +#endif + +#endif /* LIST_H */ diff --git a/C/zstdmt/lz4mt.h b/C/zstdmt/lz4mt.h new file mode 100644 index 00000000..245b7c99 --- /dev/null +++ b/C/zstdmt/lz4mt.h @@ -0,0 +1,162 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +/* *************************************** + * Defines + ****************************************/ + +#ifndef LZ4MT_H +#define LZ4MT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include /* size_t */ + +/* current maximum the library will accept */ +#define LZ4MT_THREAD_MAX 128 +#define LZ4MT_LEVEL_MAX 16 + +#define LZ4FMT_MAGICNUMBER 0x184D2204U +#define LZ4FMT_MAGIC_SKIPPABLE 0x184D2A50U + +/* ************************************** + * Error Handling + ****************************************/ + +extern size_t lz4mt_errcode; + +typedef enum { + LZ4MT_error_no_error, + LZ4MT_error_memory_allocation, + LZ4MT_error_read_fail, + LZ4MT_error_write_fail, + LZ4MT_error_data_error, + LZ4MT_error_frame_compress, + LZ4MT_error_frame_decompress, + LZ4MT_error_compressionParameter_unsupported, + LZ4MT_error_compression_library, + LZ4MT_error_maxCode +} LZ4MT_ErrorCode; + +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define PREFIX(name) LZ4MT_error_##name +#define ERROR(name) ((size_t)-PREFIX(name)) +extern unsigned LZ4MT_isError(size_t code); +extern const char* LZ4MT_getErrorString(size_t code); + +/* ************************************** + * Structures + ****************************************/ + +typedef struct { + void *buf; /* ptr to data */ + size_t size; /* current filled in buf */ + size_t allocated; /* length of buf */ +} LZ4MT_Buffer; + +/** + * reading and writing functions + * - you can use stdio functions or plain read/write + * - just write some wrapper on your own + * - a sample is given in 7-Zip ZS or lz4mt.c + * - the function should return -1 on error and zero on success + * - the read or written bytes will go to in->size or out->size + */ +typedef int (fn_read) (void *args, LZ4MT_Buffer * in); +typedef int (fn_write) (void *args, LZ4MT_Buffer * out); + +typedef struct { + fn_read *fn_read; + void *arg_read; + fn_write *fn_write; + void *arg_write; +} LZ4MT_RdWr_t; + +/* ************************************** + * Compression + ****************************************/ + +typedef struct LZ4MT_CCtx_s LZ4MT_CCtx; + +/** + * 1) allocate new cctx + * - return cctx or zero on error + * + * @level - 1 .. 9 + * @threads - 1 .. LZ4MT_THREAD_MAX + * @inputsize - if zero, becomes some optimal value for the level + * - if nonzero, the given value is taken + */ +LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize); + +/** + * 2) threaded compression + * - errorcheck via + */ +size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/** + * 3) get some statistic + */ +size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx); +size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx); + +/** + * 4) free cctx + * - no special return value + */ +void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx); + +/* ************************************** + * Decompression + ****************************************/ + +typedef struct LZ4MT_DCtx_s LZ4MT_DCtx; + +/** + * 1) allocate new cctx + * - return cctx or zero on error + * + * @threads - 1 .. LZ4MT_THREAD_MAX + * @ inputsize - used for single threaded standard lz4 format without skippable frames + */ +LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize); + +/** + * 2) threaded compression + * - return -1 on error + */ +size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr); + +/** + * 3) get some statistic + */ +size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx); +size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx); + +/** + * 4) free cctx + * - no special return value + */ +void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx); + +#if defined (__cplusplus) +} +#endif +#endif /* LZ4MT_H */ diff --git a/C/zstdmt/lz4mt_common.c b/C/zstdmt/lz4mt_common.c new file mode 100644 index 00000000..2dedd621 --- /dev/null +++ b/C/zstdmt/lz4mt_common.c @@ -0,0 +1,62 @@ + +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#include "lz4frame.h" +#include "lz4mt.h" + +/* will be used for lib errors */ +size_t lz4mt_errcode; + +/* **************************************** + * LZ4MT Error Management + ******************************************/ + +/** + * LZ4MT_isError() - tells if a return value is an error code + */ +unsigned LZ4MT_isError(size_t code) +{ + return (code > ERROR(maxCode)); +} + +/** + * LZ4MT_getErrorString() - give error code string from function result + */ +const char *LZ4MT_getErrorString(size_t code) +{ + if (LZ4F_isError(lz4mt_errcode)) + return LZ4F_getErrorName(lz4mt_errcode); + + static const char *notErrorCode = "Unspecified error lz4mt code"; + switch ((LZ4MT_ErrorCode)(0-code)) { + case PREFIX(no_error): + return "No error detected"; + case PREFIX(memory_allocation): + return "Allocation error : not enough memory"; + case PREFIX(read_fail): + return "Read failure"; + case PREFIX(write_fail): + return "Write failure"; + case PREFIX(data_error): + return "Malformed input"; + case PREFIX(frame_compress): + return "Could not compress frame at once"; + case PREFIX(frame_decompress): + return "Could not decompress frame at once"; + case PREFIX(compressionParameter_unsupported): + return "Compression parameter is out of bound"; + case PREFIX(compression_library): + return "Compression library reports failure"; + case PREFIX(maxCode): + default: + return notErrorCode; + } +} diff --git a/C/zstdmt/lz4mt_compress.c b/C/zstdmt/lz4mt_compress.c new file mode 100644 index 00000000..162af49d --- /dev/null +++ b/C/zstdmt/lz4mt_compress.c @@ -0,0 +1,377 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include + +#define LZ4F_DISABLE_OBSOLETE_ENUMS +#include "lz4frame.h" + +#include "mem.h" +#include "threading.h" +#include "list.h" +#include "lz4mt.h" + +/** + * multi threaded lz4 - multiple workers version + * + * - each thread works on his own + * - no main thread which does reading and then starting the work + * - needs a callback for reading / writing + * - each worker does his: + * 1) get read mutex and read some input + * 2) release read mutex and do compression + * 3) get write mutex and write result + * 4) begin with step 1 again, until no input + */ + +/* worker for compression */ +typedef struct { + LZ4MT_CCtx *ctx; + LZ4F_preferences_t zpref; + pthread_t pthread; +} cwork_t; + +struct writelist; +struct writelist { + size_t frame; + LZ4MT_Buffer out; + struct list_head node; +}; + +struct LZ4MT_CCtx_s { + + /* level: 1..22 */ + int level; + + /* threads: 1..LZ4MT_THREAD_MAX */ + int threads; + + /* should be used for read from input */ + int inputsize; + + /* statistic */ + size_t insize; + size_t outsize; + size_t curframe; + size_t frames; + + /* threading */ + cwork_t *cwork; + + /* reading input */ + pthread_mutex_t read_mutex; + fn_read *fn_read; + void *arg_read; + + /* writing output */ + pthread_mutex_t write_mutex; + fn_write *fn_write; + void *arg_write; + + /* lists for writing queue */ + struct list_head writelist_free; + struct list_head writelist_busy; + struct list_head writelist_done; +}; + +/* ************************************** + * Compression + ****************************************/ + +LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize) +{ + LZ4MT_CCtx *ctx; + int t; + + /* allocate ctx */ + ctx = (LZ4MT_CCtx *) malloc(sizeof(LZ4MT_CCtx)); + if (!ctx) + return 0; + + /* check threads value */ + if (threads < 1 || threads > LZ4MT_THREAD_MAX) + return 0; + + /* check level */ + if (level < 1 || level > LZ4MT_LEVEL_MAX) + return 0; + + /* calculate chunksize for one thread */ + if (inputsize) + ctx->inputsize = inputsize; + else + ctx->inputsize = 1024 * 64; + + /* setup ctx */ + ctx->level = level; + ctx->threads = threads; + ctx->insize = 0; + ctx->outsize = 0; + ctx->frames = 0; + ctx->curframe = 0; + + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); + + /* free -> busy -> out -> free -> ... */ + INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */ + INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */ + INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */ + + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); + if (!ctx->cwork) + goto err_cwork; + + for (t = 0; t < threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->ctx = ctx; + + /* setup preferences for that thread */ + memset(&w->zpref, 0, sizeof(LZ4F_preferences_t)); + w->zpref.compressionLevel = level; + w->zpref.frameInfo.blockMode = LZ4F_blockLinked; + w->zpref.frameInfo.contentSize = 1; + w->zpref.frameInfo.contentChecksumFlag = + LZ4F_contentChecksumEnabled; + + } + + return ctx; + + err_cwork: + free(ctx); + + return 0; +} + +/** + * pt_write - queue for compressed output + */ +static size_t pt_write(LZ4MT_CCtx * ctx, struct writelist *wl) +{ + struct list_head *entry; + + /* move the entry to the done list */ + list_move(&wl->node, &ctx->writelist_done); + + /* the entry isn't the currently needed, return... */ + if (wl->frame != ctx->curframe) + return 0; + + again: + /* check, what can be written ... */ + list_for_each(entry, &ctx->writelist_done) { + wl = list_entry(entry, struct writelist, node); + if (wl->frame == ctx->curframe) { + int rv = ctx->fn_write(ctx->arg_write, &wl->out); + if (rv == -1) + return ERROR(write_fail); + ctx->outsize += wl->out.size; + ctx->curframe++; + list_move(entry, &ctx->writelist_free); + goto again; + } + } + + return 0; +} + +static void *pt_compress(void *arg) +{ + cwork_t *w = (cwork_t *) arg; + LZ4MT_CCtx *ctx = w->ctx; + size_t result; + LZ4MT_Buffer in; + + /* inbuf is constant */ + in.size = ctx->inputsize; + in.buf = malloc(in.size); + if (!in.buf) + return (void *)ERROR(memory_allocation); + + for (;;) { + struct list_head *entry; + struct writelist *wl; + int rv; + + /* allocate space for new output */ + pthread_mutex_lock(&ctx->write_mutex); + if (!list_empty(&ctx->writelist_free)) { + /* take unused entry */ + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + wl->out.size = + LZ4F_compressFrameBound(ctx->inputsize, + &w->zpref) + 12; + list_move(entry, &ctx->writelist_busy); + } else { + /* allocate new one */ + wl = (struct writelist *) + malloc(sizeof(struct writelist)); + if (!wl) { + pthread_mutex_unlock(&ctx->write_mutex); + return (void *)ERROR(memory_allocation); + } + wl->out.size = + LZ4F_compressFrameBound(ctx->inputsize, + &w->zpref) + 12;; + wl->out.buf = malloc(wl->out.size); + if (!wl->out.buf) { + pthread_mutex_unlock(&ctx->write_mutex); + return (void *)ERROR(memory_allocation); + } + list_add(&wl->node, &ctx->writelist_busy); + } + pthread_mutex_unlock(&ctx->write_mutex); + + /* read new input */ + pthread_mutex_lock(&ctx->read_mutex); + in.size = ctx->inputsize; + rv = ctx->fn_read(ctx->arg_read, &in); + if (rv == -1) { + pthread_mutex_unlock(&ctx->read_mutex); + return (void *)ERROR(read_fail); + } + + /* eof */ + if (in.size == 0) { + free(in.buf); + pthread_mutex_unlock(&ctx->read_mutex); + + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + + goto okay; + } + ctx->insize += in.size; + wl->frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + + /* compress whole frame */ + result = + LZ4F_compressFrame((unsigned char *)wl->out.buf + 12, + wl->out.size - 12, in.buf, in.size, + &w->zpref); + if (LZ4F_isError(result)) { + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + /* user can lookup that code */ + lz4mt_errcode = result; + return (void *)ERROR(compression_library); + } + + /* write skippable frame */ + MEM_writeLE32((unsigned char *)wl->out.buf + 0, + LZ4FMT_MAGIC_SKIPPABLE); + MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4); + MEM_writeLE32((unsigned char *)wl->out.buf + 8, + (U32)result); + wl->out.size = result + 12; + + /* write result */ + pthread_mutex_lock(&ctx->write_mutex); + result = pt_write(ctx, wl); + pthread_mutex_unlock(&ctx->write_mutex); + if (LZ4MT_isError(result)) + return (void *)result; + } + + okay: + return 0; +} + +size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr) +{ + int t; + + if (!ctx) + return ERROR(compressionParameter_unsupported); + + /* init reading and writing functions */ + ctx->fn_read = rdwr->fn_read; + ctx->fn_write = rdwr->fn_write; + ctx->arg_read = rdwr->arg_read; + ctx->arg_write = rdwr->arg_write; + + /* start all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + pthread_create(&w->pthread, NULL, pt_compress, w); + } + + /* wait for all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + void *p; + pthread_join(w->pthread, &p); + if (p) + return (size_t) p; + } + + /* clean up lists */ + while (!list_empty(&ctx->writelist_free)) { + struct writelist *wl; + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + return 0; +} + +/* returns current uncompressed data size */ +size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->insize; +} + +/* returns the current compressed data size */ +size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->outsize; +} + +/* returns the current compressed frames */ +size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->curframe; +} + +void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx) +{ + if (!ctx) + return; + + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + free(ctx->cwork); + free(ctx); + ctx = 0; + + return; +} diff --git a/C/zstdmt/lz4mt_decompress.c b/C/zstdmt/lz4mt_decompress.c new file mode 100644 index 00000000..7dd4cc38 --- /dev/null +++ b/C/zstdmt/lz4mt_decompress.c @@ -0,0 +1,587 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include + +#define LZ4F_DISABLE_OBSOLETE_ENUMS +#include "lz4frame.h" + +#include "mem.h" +#include "threading.h" +#include "list.h" +#include "lz4mt.h" + +/** + * multi threaded lz4 - multiple workers version + * + * - each thread works on his own + * - no main thread which does reading and then starting the work + * - needs a callback for reading / writing + * - each worker does his: + * 1) get read mutex and read some input + * 2) release read mutex and do compression + * 3) get write mutex and write result + * 4) begin with step 1 again, until no input + */ + +/* worker for compression */ +typedef struct { + LZ4MT_DCtx *ctx; + pthread_t pthread; + LZ4MT_Buffer in; + LZ4F_decompressionContext_t dctx; +} cwork_t; + +struct writelist; +struct writelist { + size_t frame; + LZ4MT_Buffer out; + struct list_head node; +}; + +struct LZ4MT_DCtx_s { + + /* threads: 1..LZ4MT_THREAD_MAX */ + int threads; + + /* should be used for read from input */ + size_t inputsize; + + /* statistic */ + size_t insize; + size_t outsize; + size_t curframe; + size_t frames; + + /* threading */ + cwork_t *cwork; + + /* reading input */ + pthread_mutex_t read_mutex; + fn_read *fn_read; + void *arg_read; + + /* writing output */ + pthread_mutex_t write_mutex; + fn_write *fn_write; + void *arg_write; + + /* lists for writing queue */ + struct list_head writelist_free; + struct list_head writelist_busy; + struct list_head writelist_done; +}; + +/* ************************************** + * Decompression + ****************************************/ + +LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize) +{ + LZ4MT_DCtx *ctx; + int t; + + /* allocate ctx */ + ctx = (LZ4MT_DCtx *) malloc(sizeof(LZ4MT_DCtx)); + if (!ctx) + return 0; + + /* check threads value */ + if (threads < 1 || threads > LZ4MT_THREAD_MAX) + return 0; + + /* setup ctx */ + ctx->threads = threads; + ctx->insize = 0; + ctx->outsize = 0; + ctx->frames = 0; + ctx->curframe = 0; + + /* will be used for single stream only */ + if (inputsize) + ctx->inputsize = inputsize; + else + ctx->inputsize = 1024 * 64; /* 64K buffer */ + + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); + + INIT_LIST_HEAD(&ctx->writelist_free); + INIT_LIST_HEAD(&ctx->writelist_busy); + INIT_LIST_HEAD(&ctx->writelist_done); + + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); + if (!ctx->cwork) + goto err_cwork; + + for (t = 0; t < threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->ctx = ctx; + + /* setup thread work */ + LZ4F_createDecompressionContext(&w->dctx, LZ4F_VERSION); + } + + return ctx; + + err_cwork: + free(ctx); + + return 0; +} + +/** + * pt_write - queue for decompressed output + */ +static size_t pt_write(LZ4MT_DCtx * ctx, struct writelist *wl) +{ + struct list_head *entry; + + /* move the entry to the done list */ + list_move(&wl->node, &ctx->writelist_done); + again: + /* check, what can be written ... */ + list_for_each(entry, &ctx->writelist_done) { + wl = list_entry(entry, struct writelist, node); + if (wl->frame == ctx->curframe) { + int rv = ctx->fn_write(ctx->arg_write, &wl->out); + if (rv == -1) + return ERROR(write_fail); + ctx->outsize += wl->out.size; + ctx->curframe++; + list_move(entry, &ctx->writelist_free); + goto again; + } + } + + return 0; +} + +/** + * pt_read - read compressed output + */ +static size_t pt_read(LZ4MT_DCtx * ctx, LZ4MT_Buffer * in, size_t * frame) +{ + unsigned char hdrbuf[12]; + LZ4MT_Buffer hdr; + int rv; + + /* read skippable frame (8 or 12 bytes) */ + pthread_mutex_lock(&ctx->read_mutex); + + /* special case, first 4 bytes already read */ + if (ctx->frames == 0) { + hdr.buf = hdrbuf + 4; + hdr.size = 8; + rv = ctx->fn_read(ctx->arg_read, &hdr); + if (rv == -1) + goto error_read; + if (hdr.size != 8) + goto error_read; + hdr.buf = hdrbuf; + } else { + hdr.buf = hdrbuf; + hdr.size = 12; + rv = ctx->fn_read(ctx->arg_read, &hdr); + if (rv == -1) + goto error_read; + /* eof reached ? */ + if (hdr.size == 0) { + pthread_mutex_unlock(&ctx->read_mutex); + in->size = 0; + return 0; + } + if (hdr.size != 12) + goto error_read; + if (MEM_readLE32((unsigned char *)hdr.buf + 0) != + LZ4FMT_MAGIC_SKIPPABLE) + goto error_data; + } + + /* check header data */ + if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4) + goto error_data; + + ctx->insize += 12; + /* read new inputsize */ + { + size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8); + if (in->allocated < toRead) { + /* need bigger input buffer */ + if (in->allocated) + in->buf = realloc(in->buf, toRead); + else + in->buf = malloc(toRead); + if (!in->buf) + goto error_nomem; + in->allocated = toRead; + } + + in->size = toRead; + rv = ctx->fn_read(ctx->arg_read, in); + /* generic read failure! */ + if (rv == -1) + goto error_read; + /* needed more bytes! */ + if (in->size != toRead) + goto error_data; + + ctx->insize += in->size; + } + *frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + + /* done, no error */ + return 0; + + error_data: + pthread_mutex_unlock(&ctx->read_mutex); + return ERROR(data_error); + error_read: + pthread_mutex_unlock(&ctx->read_mutex); + return ERROR(read_fail); + error_nomem: + pthread_mutex_unlock(&ctx->read_mutex); + return ERROR(memory_allocation); +} + +static void *pt_decompress(void *arg) +{ + cwork_t *w = (cwork_t *) arg; + LZ4MT_Buffer *in = &w->in; + LZ4MT_DCtx *ctx = w->ctx; + size_t result = 0; + struct writelist *wl; + + for (;;) { + struct list_head *entry; + LZ4MT_Buffer *out; + + /* allocate space for new output */ + pthread_mutex_lock(&ctx->write_mutex); + if (!list_empty(&ctx->writelist_free)) { + /* take unused entry */ + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + list_move(entry, &ctx->writelist_busy); + } else { + /* allocate new one */ + wl = (struct writelist *) + malloc(sizeof(struct writelist)); + if (!wl) { + result = ERROR(memory_allocation); + goto error_unlock; + } + wl->out.buf = 0; + wl->out.size = 0; + wl->out.allocated = 0; + list_add(&wl->node, &ctx->writelist_busy); + } + pthread_mutex_unlock(&ctx->write_mutex); + out = &wl->out; + + /* zero should not happen here! */ + result = pt_read(ctx, in, &wl->frame); + if (LZ4MT_isError(result)) { + list_move(&wl->node, &ctx->writelist_free); + goto error_lock; + } + + if (in->size == 0) + break; + + { + /* get frame size for output buffer */ + unsigned char *src = (unsigned char *)in->buf + 6; + out->size = (size_t) MEM_readLE64(src); + } + + if (out->allocated < out->size) { + if (out->allocated) + out->buf = realloc(out->buf, out->size); + else + out->buf = malloc(out->size); + if (!out->buf) { + result = ERROR(memory_allocation); + goto error_lock; + } + out->allocated = out->size; + } + + result = + LZ4F_decompress(w->dctx, out->buf, &out->size, + in->buf, &in->size, 0); + + if (LZ4F_isError(result)) { + lz4mt_errcode = result; + result = ERROR(compression_library); + goto error_lock; + } + + if (result != 0) { + result = ERROR(frame_decompress); + goto error_lock; + } + + /* write result */ + pthread_mutex_lock(&ctx->write_mutex); + result = pt_write(ctx, wl); + if (LZ4MT_isError(result)) + goto error_unlock; + pthread_mutex_unlock(&ctx->write_mutex); + } + + /* everything is okay */ + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + if (in->allocated) + free(in->buf); + return 0; + + error_lock: + pthread_mutex_lock(&ctx->write_mutex); + error_unlock: + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + if (in->allocated) + free(in->buf); + return (void *)result; +} + +/* single threaded */ +static size_t st_decompress(void *arg) +{ + LZ4MT_DCtx *ctx = (LZ4MT_DCtx *) arg; + LZ4F_errorCode_t nextToLoad = 0; + cwork_t *w = &ctx->cwork[0]; + LZ4MT_Buffer Out; + LZ4MT_Buffer *out = &Out; + LZ4MT_Buffer *in = &w->in; + void *magic = in->buf; + size_t pos = 0; + int rv; + + /* allocate space for input buffer */ + in->size = ctx->inputsize; + in->buf = malloc(in->size); + if (!in->buf) + return ERROR(memory_allocation); + + /* allocate space for output buffer */ + out->size = ctx->inputsize; + out->buf = malloc(out->size); + if (!out->buf) { + free(in->buf); + return ERROR(memory_allocation); + } + + /* we have read already 4 bytes */ + in->size = 4; + memcpy(in->buf, magic, in->size); + + nextToLoad = + LZ4F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0); + if (LZ4F_isError(nextToLoad)) { + free(in->buf); + free(out->buf); + return ERROR(compression_library); + } + + for (; nextToLoad; pos = 0) { + if (nextToLoad > ctx->inputsize) + nextToLoad = ctx->inputsize; + + /* read new input */ + in->size = nextToLoad; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) { + free(in->buf); + free(out->buf); + return ERROR(read_fail); + } + + /* done, eof reached */ + if (in->size == 0) + break; + + /* still to read, or still to flush */ + while ((pos < in->size) || (out->size == ctx->inputsize)) { + size_t remaining = in->size - pos; + out->size = ctx->inputsize; + + /* decompress */ + nextToLoad = + LZ4F_decompress(w->dctx, out->buf, &out->size, + (unsigned char *)in->buf + pos, + &remaining, NULL); + if (LZ4F_isError(nextToLoad)) { + free(in->buf); + free(out->buf); + return ERROR(compression_library); + } + + /* have some output */ + if (out->size) { + rv = ctx->fn_write(ctx->arg_write, out); + if (rv == -1) { + free(in->buf); + free(out->buf); + return ERROR(write_fail); + } + } + + if (nextToLoad == 0) + break; + + pos += remaining; + } + } + + /* no error */ + free(out->buf); + free(in->buf); + return 0; +} + +size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr) +{ + unsigned char buf[4]; + int t, rv; + cwork_t *w = &ctx->cwork[0]; + LZ4MT_Buffer *in = &w->in; + + if (!ctx) + return ERROR(compressionParameter_unsupported); + + /* init reading and writing functions */ + ctx->fn_read = rdwr->fn_read; + ctx->fn_write = rdwr->fn_write; + ctx->arg_read = rdwr->arg_read; + ctx->arg_write = rdwr->arg_write; + + /* check for LZ4FMT_MAGIC_SKIPPABLE */ + in->buf = buf; + in->size = 4; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) + return ERROR(read_fail); + if (in->size != 4) + return ERROR(data_error); + + /* single threaded with unknown sizes */ + if (MEM_readLE32(buf) != LZ4FMT_MAGIC_SKIPPABLE) { + + /* look for correct magic */ + if (MEM_readLE32(buf) != LZ4FMT_MAGICNUMBER) + return ERROR(data_error); + + /* decompress single threaded */ + return st_decompress(ctx); + } + + /* mark unused */ + in->buf = 0; + in->size = 0; + in->allocated = 0; + + /* single threaded, but with known sizes */ + if (ctx->threads == 1) { + /* no pthread_create() needed! */ + void *p = pt_decompress(w); + if (p) + return (size_t) p; + goto okay; + } + + /* multi threaded */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->in.buf = 0; + w->in.size = 0; + w->in.allocated = 0; + pthread_create(&w->pthread, NULL, pt_decompress, w); + } + + /* wait for all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + void *p; + pthread_join(w->pthread, &p); + if (p) + return (size_t) p; + } + + okay: + /* clean up the buffers */ + while (!list_empty(&ctx->writelist_free)) { + struct writelist *wl; + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + return 0; +} + +/* returns current uncompressed data size */ +size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->insize; +} + +/* returns the current compressed data size */ +size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->outsize; +} + +/* returns the current compressed frames */ +size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->curframe; +} + +void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx) +{ + int t; + + if (!ctx) + return; + + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + LZ4F_freeDecompressionContext(w->dctx); + } + + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + free(ctx->cwork); + free(ctx); + ctx = 0; + + return; +} diff --git a/C/zstdmt/lz5mt.h b/C/zstdmt/lz5mt.h new file mode 100644 index 00000000..858717bf --- /dev/null +++ b/C/zstdmt/lz5mt.h @@ -0,0 +1,162 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +/* *************************************** + * Defines + ****************************************/ + +#ifndef LZ5MT_H +#define LZ5MT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include /* size_t */ + +/* current maximum the library will accept */ +#define LZ5MT_THREAD_MAX 128 +#define LZ5MT_LEVEL_MAX 16 + +#define LZ5FMT_MAGICNUMBER 0x184D2205U +#define LZ5FMT_MAGIC_SKIPPABLE 0x184D2A50U + +/* ************************************** + * Error Handling + ****************************************/ + +extern size_t lz5mt_errcode; + +typedef enum { + LZ5MT_error_no_error, + LZ5MT_error_memory_allocation, + LZ5MT_error_read_fail, + LZ5MT_error_write_fail, + LZ5MT_error_data_error, + LZ5MT_error_frame_compress, + LZ5MT_error_frame_decompress, + LZ5MT_error_compressionParameter_unsupported, + LZ5MT_error_compression_library, + LZ5MT_error_maxCode +} LZ5MT_ErrorCode; + +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define PREFIX(name) LZ5MT_error_##name +#define ERROR(name) ((size_t)-PREFIX(name)) +extern unsigned LZ5MT_isError(size_t code); +extern const char* LZ5MT_getErrorString(size_t code); + +/* ************************************** + * Structures + ****************************************/ + +typedef struct { + void *buf; /* ptr to data */ + size_t size; /* current filled in buf */ + size_t allocated; /* length of buf */ +} LZ5MT_Buffer; + +/** + * reading and writing functions + * - you can use stdio functions or plain read/write + * - just write some wrapper on your own + * - a sample is given in 7-Zip ZS + */ +typedef int (fn_read) (void *args, LZ5MT_Buffer * in); +typedef int (fn_write) (void *args, LZ5MT_Buffer * out); + +typedef struct { + fn_read *fn_read; + void *arg_read; + fn_write *fn_write; + void *arg_write; +} LZ5MT_RdWr_t; + +/* ************************************** + * Compression + ****************************************/ + +typedef struct LZ5MT_CCtx_s LZ5MT_CCtx; + +/** + * 1) allocate new cctx + * - return cctx or zero on error + * + * @level - 1 .. 16 + * @threads - 1 .. LZ5MT_THREAD_MAX + * @inputsize - if zero, becomes some optimal value for the level + * - if nonzero, the given value is taken + */ +LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize); + +/** + * 2) threaded compression + * - return -1 on error + */ +size_t LZ5MT_CompressCCtx(LZ5MT_CCtx * ctx, LZ5MT_RdWr_t * rdwr); + +/** + * 3) get some statistic + */ +size_t LZ5MT_GetFramesCCtx(LZ5MT_CCtx * ctx); +size_t LZ5MT_GetInsizeCCtx(LZ5MT_CCtx * ctx); +size_t LZ5MT_GetOutsizeCCtx(LZ5MT_CCtx * ctx); + +/** + * 4) free cctx + * - no special return value + */ +void LZ5MT_freeCCtx(LZ5MT_CCtx * ctx); + +/* ************************************** + * Decompression - TODO, but it's easy... + ****************************************/ + +typedef struct LZ5MT_DCtx_s LZ5MT_DCtx; + +/** + * 1) allocate new cctx + * - return cctx or zero on error + * + * @level - 1 .. 22 + * @threads - 1 .. LZ5MT_THREAD_MAX + * @srclen - the max size of src for LZ5MT_CompressCCtx() + * @dstlen - the min size of dst + */ +LZ5MT_DCtx *LZ5MT_createDCtx(int threads, int inputsize); + +/** + * 2) threaded compression + * - return -1 on error + */ +size_t LZ5MT_DecompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr); + +/** + * 3) get some statistic + */ +size_t LZ5MT_GetFramesDCtx(LZ5MT_DCtx * ctx); +size_t LZ5MT_GetInsizeDCtx(LZ5MT_DCtx * ctx); +size_t LZ5MT_GetOutsizeDCtx(LZ5MT_DCtx * ctx); + +/** + * 4) free cctx + * - no special return value + */ +void LZ5MT_freeDCtx(LZ5MT_DCtx * ctx); + +#if defined (__cplusplus) +} +#endif +#endif /* LZ5MT_H */ diff --git a/C/zstdmt/lz5mt_common.c b/C/zstdmt/lz5mt_common.c new file mode 100644 index 00000000..bb7ca23c --- /dev/null +++ b/C/zstdmt/lz5mt_common.c @@ -0,0 +1,62 @@ + +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#include "lz5frame.h" +#include "lz5mt.h" + +/* will be used for lib errors */ +size_t lz5mt_errcode; + +/* **************************************** + * LZ5MT Error Management + ******************************************/ + +/** + * LZ5MT_isError() - tells if a return value is an error code + */ +unsigned LZ5MT_isError(size_t code) +{ + return (code > ERROR(maxCode)); +} + +/** + * LZ5MT_getErrorString() - give error code string from function result + */ +const char *LZ5MT_getErrorString(size_t code) +{ + if (LZ5F_isError(lz5mt_errcode)) + return LZ5F_getErrorName(lz5mt_errcode); + + static const char *notErrorCode = "Unspecified error lz4mt code"; + switch ((LZ5MT_ErrorCode)(0-code)) { + case PREFIX(no_error): + return "No error detected"; + case PREFIX(memory_allocation): + return "Allocation error : not enough memory"; + case PREFIX(read_fail): + return "Read failure"; + case PREFIX(write_fail): + return "Write failure"; + case PREFIX(data_error): + return "Malformed input"; + case PREFIX(frame_compress): + return "Could not compress frame at once"; + case PREFIX(frame_decompress): + return "Could not decompress frame at once"; + case PREFIX(compressionParameter_unsupported): + return "Compression parameter is out of bound"; + case PREFIX(compression_library): + return "Compression library reports failure"; + case PREFIX(maxCode): + default: + return notErrorCode; + } +} diff --git a/C/zstdmt/lz5mt_compress.c b/C/zstdmt/lz5mt_compress.c new file mode 100644 index 00000000..30d0a347 --- /dev/null +++ b/C/zstdmt/lz5mt_compress.c @@ -0,0 +1,377 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include + +#define LZ5F_DISABLE_OBSOLETE_ENUMS +#include "lz5frame.h" + +#include "mem.h" +#include "threading.h" +#include "list.h" +#include "lz5mt.h" + +/** + * multi threaded lz5 - multiple workers version + * + * - each thread works on his own + * - no main thread which does reading and then starting the work + * - needs a callback for reading / writing + * - each worker does his: + * 1) get read mutex and read some input + * 2) release read mutex and do compression + * 3) get write mutex and write result + * 4) begin with step 1 again, until no input + */ + +/* worker for compression */ +typedef struct { + LZ5MT_CCtx *ctx; + LZ5F_preferences_t zpref; + pthread_t pthread; +} cwork_t; + +struct writelist; +struct writelist { + size_t frame; + LZ5MT_Buffer out; + struct list_head node; +}; + +struct LZ5MT_CCtx_s { + + /* level: 1..22 */ + int level; + + /* threads: 1..LZ5MT_THREAD_MAX */ + int threads; + + /* should be used for read from input */ + int inputsize; + + /* statistic */ + size_t insize; + size_t outsize; + size_t curframe; + size_t frames; + + /* threading */ + cwork_t *cwork; + + /* reading input */ + pthread_mutex_t read_mutex; + fn_read *fn_read; + void *arg_read; + + /* writing output */ + pthread_mutex_t write_mutex; + fn_write *fn_write; + void *arg_write; + + /* lists for writing queue */ + struct list_head writelist_free; + struct list_head writelist_busy; + struct list_head writelist_done; +}; + +/* ************************************** + * Compression + ****************************************/ + +LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize) +{ + LZ5MT_CCtx *ctx; + int t; + + /* allocate ctx */ + ctx = (LZ5MT_CCtx *) malloc(sizeof(LZ5MT_CCtx)); + if (!ctx) + return 0; + + /* check threads value */ + if (threads < 1 || threads > LZ5MT_THREAD_MAX) + return 0; + + /* check level */ + if (level < 1 || level > LZ5MT_LEVEL_MAX) + return 0; + + /* calculate chunksize for one thread */ + if (inputsize) + ctx->inputsize = inputsize; + else + ctx->inputsize = 1024 * 64; + + /* setup ctx */ + ctx->level = level; + ctx->threads = threads; + ctx->insize = 0; + ctx->outsize = 0; + ctx->frames = 0; + ctx->curframe = 0; + + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); + + /* free -> busy -> out -> free -> ... */ + INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */ + INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */ + INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */ + + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); + if (!ctx->cwork) + goto err_cwork; + + for (t = 0; t < threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->ctx = ctx; + + /* setup preferences for that thread */ + memset(&w->zpref, 0, sizeof(LZ5F_preferences_t)); + w->zpref.compressionLevel = level; + w->zpref.frameInfo.blockMode = LZ5F_blockLinked; + w->zpref.frameInfo.contentSize = 1; + w->zpref.frameInfo.contentChecksumFlag = + LZ5F_contentChecksumEnabled; + + } + + return ctx; + + err_cwork: + free(ctx); + + return 0; +} + +/** + * pt_write - queue for compressed output + */ +static size_t pt_write(LZ5MT_CCtx * ctx, struct writelist *wl) +{ + struct list_head *entry; + + /* move the entry to the done list */ + list_move(&wl->node, &ctx->writelist_done); + + /* the entry isn't the currently needed, return... */ + if (wl->frame != ctx->curframe) + return 0; + + again: + /* check, what can be written ... */ + list_for_each(entry, &ctx->writelist_done) { + wl = list_entry(entry, struct writelist, node); + if (wl->frame == ctx->curframe) { + int rv = ctx->fn_write(ctx->arg_write, &wl->out); + if (rv == -1) + return ERROR(write_fail); + ctx->outsize += wl->out.size; + ctx->curframe++; + list_move(entry, &ctx->writelist_free); + goto again; + } + } + + return 0; +} + +static void *pt_compress(void *arg) +{ + cwork_t *w = (cwork_t *) arg; + LZ5MT_CCtx *ctx = w->ctx; + size_t result; + LZ5MT_Buffer in; + + /* inbuf is constant */ + in.size = ctx->inputsize; + in.buf = malloc(in.size); + if (!in.buf) + return (void *)ERROR(memory_allocation); + + for (;;) { + struct list_head *entry; + struct writelist *wl; + int rv; + + /* allocate space for new output */ + pthread_mutex_lock(&ctx->write_mutex); + if (!list_empty(&ctx->writelist_free)) { + /* take unused entry */ + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + wl->out.size = + LZ5F_compressFrameBound(ctx->inputsize, + &w->zpref) + 12; + list_move(entry, &ctx->writelist_busy); + } else { + /* allocate new one */ + wl = (struct writelist *) + malloc(sizeof(struct writelist)); + if (!wl) { + pthread_mutex_unlock(&ctx->write_mutex); + return (void *)ERROR(memory_allocation); + } + wl->out.size = + LZ5F_compressFrameBound(ctx->inputsize, + &w->zpref) + 12;; + wl->out.buf = malloc(wl->out.size); + if (!wl->out.buf) { + pthread_mutex_unlock(&ctx->write_mutex); + return (void *)ERROR(memory_allocation); + } + list_add(&wl->node, &ctx->writelist_busy); + } + pthread_mutex_unlock(&ctx->write_mutex); + + /* read new input */ + pthread_mutex_lock(&ctx->read_mutex); + in.size = ctx->inputsize; + rv = ctx->fn_read(ctx->arg_read, &in); + if (rv == -1) { + pthread_mutex_unlock(&ctx->read_mutex); + return (void *)ERROR(read_fail); + } + + /* eof */ + if (in.size == 0) { + free(in.buf); + pthread_mutex_unlock(&ctx->read_mutex); + + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + + goto okay; + } + ctx->insize += in.size; + wl->frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + + /* compress whole frame */ + result = + LZ5F_compressFrame((unsigned char *)wl->out.buf + 12, + wl->out.size - 12, in.buf, in.size, + &w->zpref); + if (LZ5F_isError(result)) { + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + /* user can lookup that code */ + lz5mt_errcode = result; + return (void *)ERROR(compression_library); + } + + /* write skippable frame */ + MEM_writeLE32((unsigned char *)wl->out.buf + 0, + LZ5FMT_MAGIC_SKIPPABLE); + MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4); + MEM_writeLE32((unsigned char *)wl->out.buf + 8, + (U32)result); + wl->out.size = result + 12; + + /* write result */ + pthread_mutex_lock(&ctx->write_mutex); + result = pt_write(ctx, wl); + pthread_mutex_unlock(&ctx->write_mutex); + if (LZ5MT_isError(result)) + return (void *)result; + } + + okay: + return 0; +} + +size_t LZ5MT_CompressCCtx(LZ5MT_CCtx * ctx, LZ5MT_RdWr_t * rdwr) +{ + int t; + + if (!ctx) + return ERROR(compressionParameter_unsupported); + + /* init reading and writing functions */ + ctx->fn_read = rdwr->fn_read; + ctx->fn_write = rdwr->fn_write; + ctx->arg_read = rdwr->arg_read; + ctx->arg_write = rdwr->arg_write; + + /* start all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + pthread_create(&w->pthread, NULL, pt_compress, w); + } + + /* wait for all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + void *p; + pthread_join(w->pthread, &p); + if (p) + return (size_t) p; + } + + /* clean up lists */ + while (!list_empty(&ctx->writelist_free)) { + struct writelist *wl; + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + return 0; +} + +/* returns current uncompressed data size */ +size_t LZ5MT_GetInsizeCCtx(LZ5MT_CCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->insize; +} + +/* returns the current compressed data size */ +size_t LZ5MT_GetOutsizeCCtx(LZ5MT_CCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->outsize; +} + +/* returns the current compressed frames */ +size_t LZ5MT_GetFramesCCtx(LZ5MT_CCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->curframe; +} + +void LZ5MT_freeCCtx(LZ5MT_CCtx * ctx) +{ + if (!ctx) + return; + + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + free(ctx->cwork); + free(ctx); + ctx = 0; + + return; +} diff --git a/C/zstdmt/lz5mt_decompress.c b/C/zstdmt/lz5mt_decompress.c new file mode 100644 index 00000000..79a76f87 --- /dev/null +++ b/C/zstdmt/lz5mt_decompress.c @@ -0,0 +1,588 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include + +#define LZ5F_DISABLE_OBSOLETE_ENUMS +#include "lz5frame.h" + +#include "mem.h" +#include "threading.h" +#include "list.h" +#include "lz5mt.h" + +/** + * multi threaded lz5 - multiple workers version + * + * - each thread works on his own + * - no main thread which does reading and then starting the work + * - needs a callback for reading / writing + * - each worker does his: + * 1) get read mutex and read some input + * 2) release read mutex and do compression + * 3) get write mutex and write result + * 4) begin with step 1 again, until no input + */ + +/* worker for compression */ +typedef struct { + LZ5MT_DCtx *ctx; + pthread_t pthread; + LZ5MT_Buffer in; + LZ5F_decompressionContext_t dctx; +} cwork_t; + +struct writelist; +struct writelist { + size_t frame; + LZ5MT_Buffer out; + struct list_head node; +}; + +struct LZ5MT_DCtx_s { + + /* threads: 1..LZ5MT_THREAD_MAX */ + int threads; + + /* should be used for read from input */ + size_t inputsize; + + /* statistic */ + size_t insize; + size_t outsize; + size_t curframe; + size_t frames; + + /* threading */ + cwork_t *cwork; + + /* reading input */ + pthread_mutex_t read_mutex; + fn_read *fn_read; + void *arg_read; + + /* writing output */ + pthread_mutex_t write_mutex; + fn_write *fn_write; + void *arg_write; + + /* lists for writing queue */ + struct list_head writelist_free; + struct list_head writelist_busy; + struct list_head writelist_done; +}; + +/* ************************************** + * Decompression + ****************************************/ + +LZ5MT_DCtx *LZ5MT_createDCtx(int threads, int inputsize) +{ + LZ5MT_DCtx *ctx; + int t; + + /* allocate ctx */ + ctx = (LZ5MT_DCtx *) malloc(sizeof(LZ5MT_DCtx)); + if (!ctx) + return 0; + + /* check threads value */ + if (threads < 1 || threads > LZ5MT_THREAD_MAX) + return 0; + + /* setup ctx */ + ctx->threads = threads; + ctx->insize = 0; + ctx->outsize = 0; + ctx->frames = 0; + ctx->curframe = 0; + + /* will be used for single stream only */ + if (inputsize) + ctx->inputsize = inputsize; + else + ctx->inputsize = 1024 * 64; /* 64K buffer */ + + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); + + INIT_LIST_HEAD(&ctx->writelist_free); + INIT_LIST_HEAD(&ctx->writelist_busy); + INIT_LIST_HEAD(&ctx->writelist_done); + + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); + if (!ctx->cwork) + goto err_cwork; + + for (t = 0; t < threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->ctx = ctx; + + /* setup thread work */ + LZ5F_createDecompressionContext(&w->dctx, LZ5F_VERSION); + } + + return ctx; + + err_cwork: + free(ctx); + + return 0; +} + +/** + * pt_write - queue for decompressed output + */ +static size_t pt_write(LZ5MT_DCtx * ctx, struct writelist *wl) +{ + struct list_head *entry; + + /* move the entry to the done list */ + list_move(&wl->node, &ctx->writelist_done); + again: + /* check, what can be written ... */ + list_for_each(entry, &ctx->writelist_done) { + wl = list_entry(entry, struct writelist, node); + if (wl->frame == ctx->curframe) { + int rv = ctx->fn_write(ctx->arg_write, &wl->out); + if (rv == -1) + return ERROR(write_fail); + ctx->outsize += wl->out.size; + ctx->curframe++; + list_move(entry, &ctx->writelist_free); + goto again; + } + } + + return 0; +} + +/** + * pt_read - read compressed output + */ +static size_t pt_read(LZ5MT_DCtx * ctx, LZ5MT_Buffer * in, size_t * frame) +{ + unsigned char hdrbuf[12]; + LZ5MT_Buffer hdr; + int rv; + + /* read skippable frame (8 or 12 bytes) */ + pthread_mutex_lock(&ctx->read_mutex); + + /* special case, first 4 bytes already read */ + if (ctx->frames == 0) { + hdr.buf = hdrbuf + 4; + hdr.size = 8; + rv = ctx->fn_read(ctx->arg_read, &hdr); + if (rv == -1) + goto error_read; + if (hdr.size != 8) + goto error_read; + hdr.buf = hdrbuf; + } else { + hdr.buf = hdrbuf; + hdr.size = 12; + rv = ctx->fn_read(ctx->arg_read, &hdr); + if (rv == -1) + goto error_read; + /* eof reached ? */ + if (hdr.size == 0) { + pthread_mutex_unlock(&ctx->read_mutex); + in->size = 0; + return 0; + } + if (hdr.size != 12) + goto error_read; + if (MEM_readLE32((unsigned char *)hdr.buf + 0) != + LZ5FMT_MAGIC_SKIPPABLE) + goto error_data; + } + + /* check header data */ + if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4) + goto error_data; + + ctx->insize += 12; + /* read new inputsize */ + { + size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8); + if (in->allocated < toRead) { + /* need bigger input buffer */ + if (in->allocated) + in->buf = realloc(in->buf, toRead); + else + in->buf = malloc(toRead); + if (!in->buf) + goto error_nomem; + in->allocated = toRead; + } + + in->size = toRead; + rv = ctx->fn_read(ctx->arg_read, in); + /* generic read failure! */ + if (rv == -1) + goto error_read; + /* needed more bytes! */ + if (in->size != toRead) + goto error_data; + + ctx->insize += in->size; + } + *frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + + /* done, no error */ + return 0; + + error_data: + pthread_mutex_unlock(&ctx->read_mutex); + return ERROR(data_error); + error_read: + pthread_mutex_unlock(&ctx->read_mutex); + return ERROR(read_fail); + error_nomem: + pthread_mutex_unlock(&ctx->read_mutex); + return ERROR(memory_allocation); +} + +static void *pt_decompress(void *arg) +{ + cwork_t *w = (cwork_t *) arg; + LZ5MT_Buffer *in = &w->in; + LZ5MT_DCtx *ctx = w->ctx; + size_t result = 0; + struct writelist *wl; + + for (;;) { + struct list_head *entry; + LZ5MT_Buffer *out; + + /* allocate space for new output */ + pthread_mutex_lock(&ctx->write_mutex); + if (!list_empty(&ctx->writelist_free)) { + /* take unused entry */ + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + list_move(entry, &ctx->writelist_busy); + } else { + /* allocate new one */ + wl = (struct writelist *) + malloc(sizeof(struct writelist)); + if (!wl) { + result = ERROR(memory_allocation); + goto error_unlock; + } + wl->out.buf = 0; + wl->out.size = 0; + wl->out.allocated = 0; + list_add(&wl->node, &ctx->writelist_busy); + } + pthread_mutex_unlock(&ctx->write_mutex); + out = &wl->out; + + /* zero should not happen here! */ + result = pt_read(ctx, in, &wl->frame); + if (in->size == 0) + break; + + if (LZ5MT_isError(result)) { + list_move(&wl->node, &ctx->writelist_free); + + goto error_lock; + } + + { + /* get frame size for output buffer */ + unsigned char *src = (unsigned char *)in->buf + 6; + out->size = (size_t) MEM_readLE64(src); + } + + if (out->allocated < out->size) { + if (out->allocated) + out->buf = realloc(out->buf, out->size); + else + out->buf = malloc(out->size); + if (!out->buf) { + result = ERROR(memory_allocation); + goto error_lock; + } + out->allocated = out->size; + } + + result = + LZ5F_decompress(w->dctx, out->buf, &out->size, + in->buf, &in->size, 0); + + if (LZ5F_isError(result)) { + lz5mt_errcode = result; + result = ERROR(compression_library); + goto error_lock; + } + + if (result != 0) { + result = ERROR(frame_decompress); + goto error_lock; + } + + /* write result */ + pthread_mutex_lock(&ctx->write_mutex); + result = pt_write(ctx, wl); + if (LZ5MT_isError(result)) + goto error_unlock; + pthread_mutex_unlock(&ctx->write_mutex); + } + + /* everything is okay */ + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + if (in->allocated) + free(in->buf); + return 0; + + error_lock: + pthread_mutex_lock(&ctx->write_mutex); + error_unlock: + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + if (in->allocated) + free(in->buf); + return (void *)result; +} + +/* single threaded */ +static size_t st_decompress(void *arg) +{ + LZ5MT_DCtx *ctx = (LZ5MT_DCtx *) arg; + LZ5F_errorCode_t nextToLoad = 0; + cwork_t *w = &ctx->cwork[0]; + LZ5MT_Buffer Out; + LZ5MT_Buffer *out = &Out; + LZ5MT_Buffer *in = &w->in; + void *magic = in->buf; + size_t pos = 0; + int rv; + + /* allocate space for input buffer */ + in->size = ctx->inputsize; + in->buf = malloc(in->size); + if (!in->buf) + return ERROR(memory_allocation); + + /* allocate space for output buffer */ + out->size = ctx->inputsize; + out->buf = malloc(out->size); + if (!out->buf) { + free(in->buf); + return ERROR(memory_allocation); + } + + /* we have read already 4 bytes */ + in->size = 4; + memcpy(in->buf, magic, in->size); + + nextToLoad = + LZ5F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0); + if (LZ5F_isError(nextToLoad)) { + free(in->buf); + free(out->buf); + return ERROR(compression_library); + } + + for (; nextToLoad; pos = 0) { + if (nextToLoad > ctx->inputsize) + nextToLoad = ctx->inputsize; + + /* read new input */ + in->size = nextToLoad; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) { + free(in->buf); + free(out->buf); + return ERROR(read_fail); + } + + /* done, eof reached */ + if (in->size == 0) + break; + + /* still to read, or still to flush */ + while ((pos < in->size) || (out->size == ctx->inputsize)) { + size_t remaining = in->size - pos; + out->size = ctx->inputsize; + + /* decompress */ + nextToLoad = + LZ5F_decompress(w->dctx, out->buf, &out->size, + (unsigned char *)in->buf + pos, + &remaining, NULL); + if (LZ5F_isError(nextToLoad)) { + free(in->buf); + free(out->buf); + return ERROR(compression_library); + } + + /* have some output */ + if (out->size) { + rv = ctx->fn_write(ctx->arg_write, out); + if (rv == -1) { + free(in->buf); + free(out->buf); + return ERROR(write_fail); + } + } + + if (nextToLoad == 0) + break; + + pos += remaining; + } + } + + /* no error */ + free(out->buf); + free(in->buf); + return 0; +} + +size_t LZ5MT_DecompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr) +{ + unsigned char buf[4]; + int t, rv; + cwork_t *w = &ctx->cwork[0]; + LZ5MT_Buffer *in = &w->in; + + if (!ctx) + return ERROR(compressionParameter_unsupported); + + /* init reading and writing functions */ + ctx->fn_read = rdwr->fn_read; + ctx->fn_write = rdwr->fn_write; + ctx->arg_read = rdwr->arg_read; + ctx->arg_write = rdwr->arg_write; + + /* check for LZ5FMT_MAGIC_SKIPPABLE */ + in->buf = buf; + in->size = 4; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) + return ERROR(read_fail); + if (in->size != 4) + return ERROR(data_error); + + /* single threaded with unknown sizes */ + if (MEM_readLE32(buf) != LZ5FMT_MAGIC_SKIPPABLE) { + + /* look for correct magic */ + if (MEM_readLE32(buf) != LZ5FMT_MAGICNUMBER) + return ERROR(data_error); + + /* decompress single threaded */ + return st_decompress(ctx); + } + + /* mark unused */ + in->buf = 0; + in->size = 0; + in->allocated = 0; + + /* single threaded, but with known sizes */ + if (ctx->threads == 1) { + /* no pthread_create() needed! */ + void *p = pt_decompress(w); + if (p) + return (size_t) p; + goto okay; + } + + /* multi threaded */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->in.buf = 0; + w->in.size = 0; + w->in.allocated = 0; + pthread_create(&w->pthread, NULL, pt_decompress, w); + } + + /* wait for all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + void *p; + pthread_join(w->pthread, &p); + if (p) + return (size_t) p; + } + + okay: + /* clean up the buffers */ + while (!list_empty(&ctx->writelist_free)) { + struct writelist *wl; + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + return 0; +} + +/* returns current uncompressed data size */ +size_t LZ5MT_GetInsizeDCtx(LZ5MT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->insize; +} + +/* returns the current compressed data size */ +size_t LZ5MT_GetOutsizeDCtx(LZ5MT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->outsize; +} + +/* returns the current compressed frames */ +size_t LZ5MT_GetFramesDCtx(LZ5MT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->curframe; +} + +void LZ5MT_freeDCtx(LZ5MT_DCtx * ctx) +{ + int t; + + if (!ctx) + return; + + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + LZ5F_freeDecompressionContext(w->dctx); + } + + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + free(ctx->cwork); + free(ctx); + ctx = 0; + + return; +} diff --git a/C/zstdmt/mem.h b/C/zstdmt/mem.h new file mode 100644 index 00000000..f9303fda --- /dev/null +++ b/C/zstdmt/mem.h @@ -0,0 +1,379 @@ + +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + +/* define likely() and unlikely() */ +#if defined(__GNUC__) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; + __pragma( pack(pop) ) +#else + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/C/zstdmt/threading.c b/C/zstdmt/threading.c new file mode 100644 index 00000000..c8882520 --- /dev/null +++ b/C/zstdmt/threading.c @@ -0,0 +1,71 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +/** + * This file will hold wrapper for systems, which do not support Pthreads + */ + +#ifdef _WIN32 + +/** + * Windows Pthread Wrapper, based on this site: + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ + +#include "threading.h" + +#include +#include + +static unsigned __stdcall worker(void *arg) +{ + pthread_t *thread = (pthread_t *) arg; + thread->arg = thread->start_routine(thread->arg); + return 0; +} + +int +pthread_create(pthread_t * thread, const void *unused, + void *(*start_routine) (void *), void *arg) +{ + (void)unused; + thread->arg = arg; + thread->start_routine = start_routine; + thread->handle = + (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); + + if (!thread->handle) + return errno; + else + return 0; +} + +int _pthread_join(pthread_t * thread, void **value_ptr) +{ + if (!thread->handle) + return 0; + + DWORD result = WaitForSingleObject(thread->handle, INFINITE); + switch (result) { + case WAIT_OBJECT_0: + if (value_ptr) + *value_ptr = thread->arg; + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif diff --git a/C/zstdmt/threading.h b/C/zstdmt/threading.h new file mode 100644 index 00000000..9e14fa4d --- /dev/null +++ b/C/zstdmt/threading.h @@ -0,0 +1,68 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#ifndef THREADING_H +#define THREADING_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifdef _WIN32 + +/** + * Windows Pthread Wrapper, based on this site: + * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include + +/* mutex */ +#define pthread_mutex_t CRITICAL_SECTION +#define pthread_mutex_init(a,b) InitializeCriticalSection((a)) +#define pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define pthread_mutex_lock EnterCriticalSection +#define pthread_mutex_unlock LeaveCriticalSection + +/* pthread_create() and pthread_join() */ +typedef struct { + HANDLE handle; + void *(*start_routine) (void *); + void *arg; +} pthread_t; + +extern int pthread_create(pthread_t * thread, const void *unused, + void *(*start_routine) (void *), void *arg); + +#define pthread_join(a, b) _pthread_join(&(a), (b)) +extern int _pthread_join(pthread_t * thread, void **value_ptr); + +/** + * add here more systems as required + */ + +#else + +/* POSIX Systems */ +#include + +#endif /* POSIX Systems */ + +#if defined (__cplusplus) +} +#endif + +#endif /* PTHREAD_H */ diff --git a/C/zstdmt/zstdmt.h b/C/zstdmt/zstdmt.h new file mode 100644 index 00000000..114c7961 --- /dev/null +++ b/C/zstdmt/zstdmt.h @@ -0,0 +1,202 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +/* *************************************** + * Defines + ****************************************/ + +#ifndef ZSTDMT_H +#define ZSTDMT_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include /* size_t */ + +#define ZSTDMT_THREAD_MAX 128 +#define ZSTDMT_LEVEL_MAX 22 + +/* zstd magic values */ +#define ZSTDMT_MAGICNUMBER_V01 0x1EB52FFDU +#define ZSTDMT_MAGICNUMBER_MIN 0xFD2FB522U +#define ZSTDMT_MAGICNUMBER_MAX 0xFD2FB528U +#define ZSTDMT_MAGIC_SKIPPABLE 0x184D2A50U + +/* ************************************** + * Error Handling + ****************************************/ + +typedef enum { + ZSTDMT_error_no_error, + ZSTDMT_error_memory_allocation, + ZSTDMT_error_init_missing, + ZSTDMT_error_read_fail, + ZSTDMT_error_write_fail, + ZSTDMT_error_data_error, + ZSTDMT_error_frame_compress, + ZSTDMT_error_frame_decompress, + ZSTDMT_error_compressionParameter_unsupported, + ZSTDMT_error_compression_library, + ZSTDMT_error_maxCode +} ZSTDMT_ErrorCode; + +extern size_t zstdmt_errcode; + +#define ZSTDMT_PREFIX(name) ZSTDMT_error_##name +#define ZSTDMT_ERROR(name) ((size_t)-ZSTDMT_PREFIX(name)) +extern unsigned ZSTDMT_isError(size_t code); +extern const char* ZSTDMT_getErrorString(size_t code); + +/* ************************************** + * Structures + ****************************************/ + +typedef struct { + void *buf; /* ptr to data */ + size_t size; /* current filled in buf */ + size_t allocated; /* length of buf */ +} ZSTDMT_Buffer; + +/** + * reading and writing functions + * - you can use stdio functions or plain read/write + * - just write some wrapper on your own + * - a sample is given in 7-Zip ZS + */ +typedef int (fn_read) (void *args, ZSTDMT_Buffer * in); +typedef int (fn_write) (void *args, ZSTDMT_Buffer * out); + +typedef struct { + fn_read *fn_read; + void *arg_read; + fn_write *fn_write; + void *arg_write; +} ZSTDMT_RdWr_t; + +/* ************************************** + * Compression + ****************************************/ + +typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; + +/** + * ZSTDMT_createCCtx() - allocate new compression context + * + * This function allocates and initializes an zstd commpression context. + * The context can be used multiple times without the need for resetting + * or re-initializing. + * + * @level: compression level, which should be used (1..22) + * @threads: number of threads, which should be used (1..ZSTDMT_THREAD_MAX) + * @inputsize: - if zero, becomes some optimal value for the level + * - if nonzero, the given value is taken + * @zstdmt_errcode: space for storing zstd errors (needed for thread safety) + * @return: the context on success, zero on error + */ +ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize); + +/** + * ZSTDMT_compressDCtx() - threaded compression for zstd + * + * This function will create valid zstd streams. The number of threads, + * the input chunksize and the compression level are .... + * + * @ctx: context, which needs to be created with ZSTDMT_createDCtx() + * @rdwr: callback structure, which defines reding/writing functions + * @return: zero on success, or error code + */ +size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx * ctx, ZSTDMT_RdWr_t * rdwr); + +/** + * ZSTDMT_GetFramesCCtx() - number of written frames + * ZSTDMT_GetInsizeCCtx() - read bytes of input + * ZSTDMT_GetOutsizeCCtx() - written bytes of output + * + * These three functions will return some statistical data of the + * compression context ctx. + * + * @ctx: context, which should be examined + * @return: the request value, or zero on error + */ +size_t ZSTDMT_GetFramesCCtx(ZSTDMT_CCtx * ctx); +size_t ZSTDMT_GetInsizeCCtx(ZSTDMT_CCtx * ctx); +size_t ZSTDMT_GetOutsizeCCtx(ZSTDMT_CCtx * ctx); + +/** + * ZSTDMT_freeCCtx() - free compression context + * + * This function will free all allocated resources, which were allocated + * by ZSTDMT_createCCtx(). This function can not fail. + * + * @ctx: context, which should be freed + */ +void ZSTDMT_freeCCtx(ZSTDMT_CCtx * ctx); + +/* ************************************** + * Decompression + ****************************************/ + +typedef struct ZSTDMT_DCtx_s ZSTDMT_DCtx; + +/** + * 1) allocate new cctx + * - return cctx or zero on error + * + * @level - 1 .. 22 + * @threads - 1 .. ZSTDMT_THREAD_MAX + * @srclen - the max size of src for ZSTDMT_compressCCtx() + * @dstlen - the min size of dst + */ +ZSTDMT_DCtx *ZSTDMT_createDCtx(int threads, int inputsize); + +/** + * ZSTDMT_decompressDCtx() - threaded decompression for zstd + * + * This function will decompress valid zstd streams. + * + * @ctx: context, which needs to be created with ZSTDMT_createDCtx() + * @rdwr: callback structure, which defines reding/writing functions + * @return: zero on success, or error code + */ +size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr); + +/** + * ZSTDMT_GetFramesDCtx() - number of read frames + * ZSTDMT_GetInsizeDCtx() - read bytes of input + * ZSTDMT_GetOutsizeDCtx() - written bytes of output + * + * These three functions will return some statistical data of the + * decompression context ctx. + * + * @ctx: context, which should be examined + * @return: the request value, or zero on error + */ +size_t ZSTDMT_GetFramesDCtx(ZSTDMT_DCtx * ctx); +size_t ZSTDMT_GetInsizeDCtx(ZSTDMT_DCtx * ctx); +size_t ZSTDMT_GetOutsizeDCtx(ZSTDMT_DCtx * ctx); + +/** + * ZSTDMT_freeDCtx() - free decompression context + * + * This function will free all allocated resources, which were allocated + * by ZSTDMT_createDCtx(). This function can not fail. + * + * @ctx: context, which should be freed + */ +void ZSTDMT_freeDCtx(ZSTDMT_DCtx * ctx); + +#if defined (__cplusplus) +} +#endif +#endif /* ZSTDMT_H */ diff --git a/C/zstdmt/zstdmt_common.c b/C/zstdmt/zstdmt_common.c new file mode 100644 index 00000000..2083dfcc --- /dev/null +++ b/C/zstdmt/zstdmt_common.c @@ -0,0 +1,61 @@ + +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#include "zstd.h" +#include "zstdmt.h" + +/* **************************************** + * ZSTDMT Error Management + ******************************************/ + +size_t zstdmt_errcode; + +/** + * ZSTDMT_isError() - tells if a return value is an error code + */ +unsigned ZSTDMT_isError(size_t code) +{ + return (code > ZSTDMT_ERROR(maxCode)); +} + +/** + * LZ4MT_getErrorString() - give error code string from function result + */ +const char *ZSTDMT_getErrorString(size_t code) +{ + if (ZSTD_isError(zstdmt_errcode)) + return ZSTD_getErrorName(zstdmt_errcode); + + static const char *notErrorCode = "Unspecified error zstmt code"; + switch ((ZSTDMT_ErrorCode) (0 - code)) { + case ZSTDMT_PREFIX(no_error): + return "No error detected"; + case ZSTDMT_PREFIX(memory_allocation): + return "Allocation error : not enough memory"; + case ZSTDMT_PREFIX(read_fail): + return "Read failure"; + case ZSTDMT_PREFIX(write_fail): + return "Write failure"; + case ZSTDMT_PREFIX(data_error): + return "Malformed input"; + case ZSTDMT_PREFIX(frame_compress): + return "Could not compress frame at once"; + case ZSTDMT_PREFIX(frame_decompress): + return "Could not decompress frame at once"; + case ZSTDMT_PREFIX(compressionParameter_unsupported): + return "Compression parameter is out of bound"; + case ZSTDMT_PREFIX(compression_library): + return "Compression library reports failure"; + case ZSTDMT_PREFIX(maxCode): + default: + return notErrorCode; + } +} diff --git a/C/zstdmt/zstdmt_compress.c b/C/zstdmt/zstdmt_compress.c new file mode 100644 index 00000000..92ea82f0 --- /dev/null +++ b/C/zstdmt/zstdmt_compress.c @@ -0,0 +1,427 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +#include "mem.h" +#include "threading.h" +#include "list.h" +#include "zstdmt.h" + +/** + * multi threaded zstd compression + * + * - each thread works on his own + * - needs a callback for reading / writing + * - each worker does this: + * 1) get read mutex and read some input + * 2) release read mutex and do compression + * 3) get write mutex and write result + * 4) begin with step 1 again, until no input + */ + +/* worker for compression */ +typedef struct { + ZSTDMT_CCtx *ctx; + pthread_t pthread; +} cwork_t; + +struct writelist; +struct writelist { + size_t frame; + ZSTDMT_Buffer out; + struct list_head node; +}; + +struct ZSTDMT_CCtx_s { + + /* level: 1..ZSTDMT_LEVEL_MAX */ + int level; + + /* threads: 1..ZSTDMT_THREAD_MAX */ + int threads; + + /* buffersize for reading input */ + int inputsize; + + /* statistic */ + size_t insize; + size_t outsize; + size_t curframe; + size_t frames; + + /* threading */ + cwork_t *cwork; + + /* reading input */ + pthread_mutex_t read_mutex; + fn_read *fn_read; + void *arg_read; + + /* writing output */ + pthread_mutex_t write_mutex; + fn_write *fn_write; + void *arg_write; + + /* error handling */ + pthread_mutex_t error_mutex; + size_t zstd_errcode; + size_t zstdmt_errcode; + + /* lists for writing queue */ + struct list_head writelist_free; + struct list_head writelist_busy; + struct list_head writelist_done; +}; + +/* ************************************** + * Compression + ****************************************/ + +ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize) +{ + ZSTDMT_CCtx *ctx; + int t; + + /* allocate ctx */ + ctx = (ZSTDMT_CCtx *) malloc(sizeof(ZSTDMT_CCtx)); + if (!ctx) + return 0; + + /* check threads value */ + if (threads < 1 || threads > ZSTDMT_THREAD_MAX) + goto err_ctx; + + /* check level */ + if (level < 1 || level > ZSTDMT_LEVEL_MAX) + goto err_ctx; + + /* calculate chunksize for one thread */ + if (inputsize) + ctx->inputsize = inputsize; + else { + /* XXX - windowlog */ const int mb[] = { + 2, 2, 4, 4, 6, 6, 6, /* 1 - 7 */ + 8, 8, 8, 8, 8, 8, 8, /* 8 - 14 */ + 16, 16, 16, 16, 16, 16, 16, 16 /* 15 - 22 */ + }; + ctx->inputsize = 1024 * 1024 * mb[level - 1]; + } + + /* setup ctx */ + ctx->level = level; + ctx->threads = threads; + + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); + pthread_mutex_init(&ctx->error_mutex, NULL); + + INIT_LIST_HEAD(&ctx->writelist_free); + INIT_LIST_HEAD(&ctx->writelist_busy); + INIT_LIST_HEAD(&ctx->writelist_done); + + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads); + if (!ctx->cwork) + goto err_ctx; + + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + w->ctx = ctx; + } + + return ctx; + +err_ctx: + free(ctx); + return 0; +} + +/** + * pt_write - queue for compressed output + */ +static size_t pt_write(ZSTDMT_CCtx * ctx, struct writelist *wl) +{ + struct list_head *entry; + int rv; + + /* move the entry to the done list */ + list_move(&wl->node, &ctx->writelist_done); + + /* write zero byte frame (9 bytes) for type identification */ + if (unlikely(wl->frame == 0)) { + unsigned char frame0[] = + { 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x48, 0x01, 0x00, 0x00 }; + ZSTDMT_Buffer b; + b.buf = frame0; + b.size = 9; + rv = ctx->fn_write(ctx->arg_write, &b); + if (rv == -1) + return ZSTDMT_ERROR(write_fail); + if (b.size != 9) + return ZSTDMT_ERROR(write_fail); + ctx->outsize += 9; + } + + /* the entry isn't the currently needed, return... */ + if (wl->frame != ctx->curframe) + return 0; + + again: + /* check, what can be written ... */ + list_for_each(entry, &ctx->writelist_done) { + wl = list_entry(entry, struct writelist, node); + if (wl->frame == ctx->curframe) { + rv = ctx->fn_write(ctx->arg_write, &wl->out); + if (rv == -1) + return ZSTDMT_ERROR(write_fail); + ctx->outsize += wl->out.size; + ctx->curframe++; + list_move(entry, &ctx->writelist_free); + goto again; + } + } + + return 0; +} + +/* parallel compression worker */ +static void *pt_compress(void *arg) +{ + cwork_t *w = (cwork_t *) arg; + ZSTDMT_CCtx *ctx = w->ctx; + struct writelist *wl; + size_t result; + ZSTDMT_Buffer in; + + /* inbuf is constant */ + in.size = ctx->inputsize; + in.buf = malloc(in.size); + if (!in.buf) + return (void *)ZSTDMT_ERROR(memory_allocation); + + for (;;) { + struct list_head *entry; + ZSTDMT_Buffer *out; + int rv; + + /* allocate space for new output */ + pthread_mutex_lock(&ctx->write_mutex); + if (!list_empty(&ctx->writelist_free)) { + /* take unused entry */ + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12; + list_move(entry, &ctx->writelist_busy); + } else { + /* allocate new one */ + wl = (struct writelist *) + malloc(sizeof(struct writelist)); + if (!wl) { + pthread_mutex_unlock(&ctx->write_mutex); + free(in.buf); + return (void *)ZSTDMT_ERROR(memory_allocation); + } + wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;; + wl->out.buf = malloc(wl->out.size); + if (!wl->out.buf) { + pthread_mutex_unlock(&ctx->write_mutex); + free(in.buf); + return (void *)ZSTDMT_ERROR(memory_allocation); + } + list_add(&wl->node, &ctx->writelist_busy); + } + pthread_mutex_unlock(&ctx->write_mutex); + out = &wl->out; + + /* read new input */ + pthread_mutex_lock(&ctx->read_mutex); + in.size = ctx->inputsize; + rv = ctx->fn_read(ctx->arg_read, &in); + if (rv == -1) { + pthread_mutex_unlock(&ctx->read_mutex); + result = ZSTDMT_ERROR(read_fail); + goto error; + } + + /* eof */ + if (in.size == 0) { + free(in.buf); + pthread_mutex_unlock(&ctx->read_mutex); + + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + + goto okay; + } + ctx->insize += in.size; + wl->frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + + /* compress whole frame */ + { + unsigned char *outbuf = out->buf; + result = ZSTD_compress(outbuf + 12, out->size - 12, in.buf, in.size, ctx->level); + if (ZSTD_isError(result)) { + zstdmt_errcode = result; + result = ZSTDMT_ERROR(compression_library); + goto error; + } + } + + /* write skippable frame */ + { + unsigned char *outbuf = out->buf; + + MEM_writeLE32(outbuf + 0, ZSTDMT_MAGIC_SKIPPABLE); + MEM_writeLE32(outbuf + 4, 4); + MEM_writeLE32(outbuf + 8, (U32) result); + out->size = result + 12; + } + + /* write result */ + pthread_mutex_lock(&ctx->write_mutex); + result = pt_write(ctx, wl); + pthread_mutex_unlock(&ctx->write_mutex); + if (ZSTDMT_isError(result)) + goto error; + } + + okay: + return 0; + error: + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + return (void *)result; +} + +/* compress data, until input ends */ +size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx * ctx, ZSTDMT_RdWr_t * rdwr) +{ + int t; + void *rv = 0; + + if (!ctx) + return ZSTDMT_ERROR(init_missing); + + /* setup reading and writing functions */ + ctx->fn_read = rdwr->fn_read; + ctx->fn_write = rdwr->fn_write; + ctx->arg_read = rdwr->arg_read; + ctx->arg_write = rdwr->arg_write; + + /* init counter and error codes */ + ctx->insize = 0; + ctx->outsize = 0; + ctx->frames = 0; + ctx->curframe = 0; + ctx->zstdmt_errcode = 0; + + /* start all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + pthread_create(&w->pthread, NULL, pt_compress, w); + } + + /* wait for all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + void *p; + pthread_join(w->pthread, &p); + if (p) + rv = p; + } + + /* clean up the free list */ + while (!list_empty(&ctx->writelist_free)) { + struct writelist *wl; + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + /* on error, these two lists may have some entries */ + if (rv) { + struct writelist *wl; + struct list_head *entry; + + while (!list_empty(&ctx->writelist_busy)) { + entry = list_first(&ctx->writelist_busy); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + while (!list_empty(&ctx->writelist_done)) { + entry = list_first(&ctx->writelist_done); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + } + + return (size_t) rv; +} + +/* returns current uncompressed data size */ +size_t ZSTDMT_GetInsizeCCtx(ZSTDMT_CCtx * ctx) +{ + if (!ctx) + return ZSTDMT_ERROR(init_missing); + + /* no mutex needed here */ + return ctx->insize; +} + +/* returns the current compressed data size */ +size_t ZSTDMT_GetOutsizeCCtx(ZSTDMT_CCtx * ctx) +{ + if (!ctx) + return ZSTDMT_ERROR(init_missing); + + /* no mutex needed here */ + return ctx->outsize; +} + +/* returns the current compressed data frame count */ +size_t ZSTDMT_GetFramesCCtx(ZSTDMT_CCtx * ctx) +{ + if (!ctx) + return ZSTDMT_ERROR(init_missing); + + /* no mutex needed here */ + return ctx->curframe; +} + +/* free all allocated buffers and structures */ +void ZSTDMT_freeCCtx(ZSTDMT_CCtx * ctx) +{ + if (!ctx) + return; + + free(ctx->cwork); + free(ctx); + ctx = 0; + + return; +} diff --git a/C/zstdmt/zstdmt_decompress.c b/C/zstdmt/zstdmt_decompress.c new file mode 100644 index 00000000..ead579bb --- /dev/null +++ b/C/zstdmt/zstdmt_decompress.c @@ -0,0 +1,857 @@ + +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + */ + +#include +#include + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +#include "mem.h" +#include "threading.h" +#include "list.h" +#include "zstdmt.h" + +/** + * multi threaded zstd decompression + * + * - each thread works on his own + * - needs a callback for reading / writing + * - each worker does this: + * 1) get read mutex and read some input + * 2) release read mutex and do decompression + * 3) get write mutex and write result + * 4) begin with step 1 again, until no input + */ + +#if 0 +#include +#define dprintf(fmt, arg...) do { printf(fmt, ## arg); } while (0) +#else +#define dprintf(fmt, ...) +#endif /* DEBUG */ + +extern size_t zstdmt_errcode; + +/* worker for compression */ +typedef struct { + ZSTDMT_DCtx *ctx; + pthread_t pthread; + ZSTDMT_Buffer in; + ZSTD_DStream *dctx; +} cwork_t; + +struct writelist; +struct writelist { + size_t frame; + ZSTDMT_Buffer out; + struct list_head node; +}; + +struct ZSTDMT_DCtx_s { + + /* threads: 1..ZSTDMT_THREAD_MAX */ + int threads; + int threadswanted; + + /* input buffer, used at single threading */ + size_t inputsize; + + /* buffersize used for output */ + size_t outputsize; + + /* statistic */ + size_t insize; + size_t outsize; + size_t curframe; + size_t frames; + + /* threading */ + cwork_t *cwork; + + /* reading input */ + pthread_mutex_t read_mutex; + fn_read *fn_read; + void *arg_read; + + /* writing output */ + pthread_mutex_t write_mutex; + fn_write *fn_write; + void *arg_write; + + /* error handling */ + pthread_mutex_t error_mutex; + + /* lists for writing queue */ + struct list_head writelist_free; + struct list_head writelist_busy; + struct list_head writelist_done; +}; + +/* ************************************** + * Decompression + ****************************************/ + +ZSTDMT_DCtx *ZSTDMT_createDCtx(int threads, int inputsize) +{ + ZSTDMT_DCtx *ctx; + + /* allocate ctx */ + ctx = (ZSTDMT_DCtx *) malloc(sizeof(ZSTDMT_DCtx)); + if (!ctx) + return 0; + + /* check threads value */ + if (threads < 1 || threads > ZSTDMT_THREAD_MAX) + return 0; + + /* setup ctx */ + ctx->threadswanted = threads; + ctx->threads = 0; + ctx->insize = 0; + ctx->outsize = 0; + ctx->frames = 0; + ctx->curframe = 0; + + /* will be used for single stream only */ + if (inputsize) + ctx->inputsize = inputsize; + else + ctx->inputsize = 1024 * 512; + + /* frame size (will get higher, when needed) */ + ctx->outputsize = 1024 * 512; + + /* later */ + ctx->cwork = 0; + + return ctx; +} + +/** + * IsZstd_Magic - check, if 4 bytes are valid ZSTD MAGIC + */ +static int IsZstd_Magic(unsigned char *buf) +{ + U32 magic = MEM_readLE32(buf); + if (magic == ZSTDMT_MAGICNUMBER_V01) + return 1; + return (magic >= ZSTDMT_MAGICNUMBER_MIN + && magic <= ZSTDMT_MAGICNUMBER_MAX); +} + +/** + * IsZstd_Skippable - check, if 4 bytes are MAGIC_SKIPPABLE + */ +static int IsZstd_Skippable(unsigned char *buf) +{ + return (MEM_readLE32(buf) == ZSTDMT_MAGIC_SKIPPABLE); +} + +/** + * pt_write - queue for decompressed output + */ +static size_t pt_write(ZSTDMT_DCtx * ctx, struct writelist *wl) +{ + struct list_head *entry; + + /* move the entry to the done list */ + list_move(&wl->node, &ctx->writelist_done); + again: + /* check, what can be written ... */ + list_for_each(entry, &ctx->writelist_done) { + wl = list_entry(entry, struct writelist, node); + if (wl->frame == ctx->curframe) { + int rv = ctx->fn_write(ctx->arg_write, &wl->out); + if (rv == -1) + return ZSTDMT_ERROR(write_fail); + ctx->outsize += wl->out.size; + ctx->curframe++; + list_move(entry, &ctx->writelist_free); + goto again; + } + } + + return 0; +} + +/** + * pt_read - read compressed input + */ +static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame) +{ + unsigned char hdrbuf[12]; + ZSTDMT_Buffer hdr; + size_t toRead; + int rv; + + pthread_mutex_lock(&ctx->read_mutex); + + /* special case, some bytes were read by magic check */ + if (unlikely(ctx->frames == 0)) { + /* the magic check reads exactly 16 bytes! */ + if (unlikely(in->size != 16)) + goto error_data; + ctx->insize += 16; + + /** + * zstdmt mode, with zstd magic prefix + * 9 bytes zero byte frame + 12 byte skippable + * - 21 bytes to read, 16 bytes done + * - read 5 bytes, put them together (12 byte hdr) + */ + if (!IsZstd_Skippable(in->buf)) { + memcpy(hdrbuf, in->buf, 7); + hdr.buf = hdrbuf + 7; + hdr.size = 5; + rv = ctx->fn_read(ctx->arg_read, &hdr); + if (rv == -1) + goto error_read; + if (hdr.size != 5) + goto error_data; + hdr.buf = hdrbuf; + ctx->insize += 16 + 5; + + /* read data */ + toRead = MEM_readLE32((unsigned char *)hdr.buf + 8); + in->size = toRead; + in->buf = malloc(in->size); + if (!in->buf) + goto error_nomem; + in->allocated = in->size; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) + goto error_read; + if (in->size != toRead) + goto error_data; + ctx->insize += in->size; + *frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + return 0; /* done! */ + } + + /** + * pzstd mode, no prefix + * - start directly with 12 byte skippable frame + */ + if (IsZstd_Skippable(in->buf)) { + unsigned char *start = in->buf; /* 16 bytes data */ + toRead = MEM_readLE32((unsigned char *)start + 8); + in->size = toRead; + in->buf = malloc(in->size); + if (!in->buf) + goto error_nomem; + in->allocated = in->size; + /* copy 4 bytes user data to new buf */ + memcpy(in->buf, start + 12, 4); + start = in->buf; /* point to in->buf now */ + + /* 12 byte skippable, so 4 bytes data done */ + in->buf = start + 4; + in->size = toRead - 4; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) + goto error_read; + if (in->size != toRead - 4) + goto error_data; + ctx->insize += in->size; + in->buf = start; /* restore inbuf */ + in->size += 4; + *frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + return 0; /* done! */ + } + } + + /** + * read next skippable frame (12 bytes) + * 4 bytes skippable magic + * 4 bytes little endian, must be: 4 (user data size) + * 4 bytes little endian, size to read (user data) + */ + hdr.buf = hdrbuf; + hdr.size = 12; + rv = ctx->fn_read(ctx->arg_read, &hdr); + if (rv == -1) + goto error_read; + + /* eof reached ? */ + if (unlikely(hdr.size == 0)) { + pthread_mutex_unlock(&ctx->read_mutex); + in->size = 0; + return 0; + } + + /* check header data */ + if (unlikely(hdr.size != 12)) + goto error_read; + if (unlikely(!IsZstd_Skippable(hdr.buf))) + goto error_data; + ctx->insize += 12; + + /* read new input (size should be _toRead_ bytes */ + toRead = MEM_readLE32((unsigned char *)hdr.buf + 8); + { + if (in->allocated < toRead) { + /* need bigger input buffer */ + if (in->allocated) + in->buf = realloc(in->buf, toRead); + else + in->buf = malloc(toRead); + if (!in->buf) + goto error_nomem; + in->allocated = toRead; + } + + in->size = toRead; + rv = ctx->fn_read(ctx->arg_read, in); + /* generic read failure! */ + if (rv == -1) + goto error_read; + /* needed more bytes! */ + if (in->size != toRead) + goto error_data; + + ctx->insize += in->size; + } + *frame = ctx->frames++; + pthread_mutex_unlock(&ctx->read_mutex); + + /* done, no error */ + return 0; + + error_data: + pthread_mutex_unlock(&ctx->read_mutex); + return ZSTDMT_ERROR(data_error); + error_read: + pthread_mutex_unlock(&ctx->read_mutex); + return ZSTDMT_ERROR(read_fail); + error_nomem: + pthread_mutex_unlock(&ctx->read_mutex); + return ZSTDMT_ERROR(memory_allocation); +} + +static void *pt_decompress(void *arg) +{ + cwork_t *w = (cwork_t *) arg; + ZSTDMT_Buffer *in = &w->in; + ZSTDMT_DCtx *ctx = w->ctx; + struct writelist *wl; + size_t result = 0; + ZSTDMT_Buffer collect; + + /* init dstream stream */ + result = ZSTD_initDStream(w->dctx); + if (ZSTD_isError(result)) { + zstdmt_errcode = result; + return (void *)ZSTDMT_ERROR(compression_library); + } + + collect.buf = 0; + collect.size = 0; + collect.allocated = 0; + for (;;) { + ZSTDMT_Buffer *out; + ZSTD_inBuffer zIn; + ZSTD_outBuffer zOut; + + /* select or allocate space for new output */ + pthread_mutex_lock(&ctx->write_mutex); + if (!list_empty(&ctx->writelist_free)) { + /* take unused entry */ + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + list_move(entry, &ctx->writelist_busy); + } else { + /* allocate new one */ + wl = (struct writelist *) + malloc(sizeof(struct writelist)); + if (!wl) { + result = ZSTDMT_ERROR(memory_allocation); + goto error_unlock; + } + out = &wl->out; + out->size = ctx->outputsize; + out->buf = malloc(out->size); + if (!out->buf) { + result = ZSTDMT_ERROR(memory_allocation); + goto error_unlock; + } + out->allocated = out->size; + list_add(&wl->node, &ctx->writelist_busy); + } + + /* start with 512KB */ + /* XXX, add framesize detection... */ + out = &wl->out; + pthread_mutex_unlock(&ctx->write_mutex); + + /* init dstream stream */ + result = ZSTD_resetDStream(w->dctx); + if (ZSTD_isError(result)) { + zstdmt_errcode = result; + return (void *)ZSTDMT_ERROR(compression_library); + } + + /* zero should not happen here! */ + result = pt_read(ctx, in, &wl->frame); + if (in->size == 0) + break; + if (ZSTDMT_isError(result)) { + goto error_lock; + } + + zIn.size = in->allocated; + zIn.src = in->buf; + zIn.pos = 0; + + for (;;) { + again: + /* decompress loop */ + zOut.size = out->allocated; + zOut.dst = out->buf; + zOut.pos = 0; + + dprintf + ("ZSTD_decompressStream() zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n", + zIn.size, zIn.pos, zOut.size, zOut.pos); + result = ZSTD_decompressStream(w->dctx, &zOut, &zIn); + dprintf + ("ZSTD_decompressStream(), ret=%zu zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n", + result, zIn.size, zIn.pos, zOut.size, zOut.pos); + if (ZSTD_isError(result)) + goto error_clib; + + /* end of frame */ + if (result == 0) { + /* put collected stuff together */ + if (collect.size) { + void *bnew; + bnew = malloc(collect.size + zOut.pos); + if (!bnew) { + result = + ZSTDMT_ERROR(memory_allocation); + goto error_lock; + } + memcpy((char *)bnew, collect.buf, + collect.size); + memcpy((char *)bnew + collect.size, + out->buf, zOut.pos); + free(collect.buf); + free(out->buf); + out->buf = bnew; + out->size = collect.size + zOut.pos; + out->allocated = out->size; + collect.buf = 0; + collect.size = 0; + } else { + out->size = zOut.pos; + } + /* write result */ + pthread_mutex_lock(&ctx->write_mutex); + result = pt_write(ctx, wl); + if (ZSTDMT_isError(result)) + goto error_unlock; + pthread_mutex_unlock(&ctx->write_mutex); + /* will read next input */ + break; + } + + /* out buffer to small for full frame */ + if (result != 0) { + /* collect old content from out */ + collect.buf = + realloc(collect.buf, + collect.size + out->size); + memcpy((char *)collect.buf + collect.size, + out->buf, out->size); + collect.size = collect.size + out->size; + + /* double the buffer, until it fits */ + pthread_mutex_lock(&ctx->write_mutex); + out->size *= 2; + ctx->outputsize = out->size; + pthread_mutex_unlock(&ctx->write_mutex); + out->buf = realloc(out->buf, out->size); + if (!out->buf) { + result = ZSTDMT_ERROR(memory_allocation); + goto error_lock; + } + out->allocated = out->size; + goto again; + } + + if (zIn.pos == zIn.size) + break; /* should fail... */ + } /* decompress loop */ + } /* read input loop */ + + /* everything is okay */ + pthread_mutex_lock(&ctx->write_mutex); + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + if (in->allocated) + free(in->buf); + return 0; + + error_clib: + zstdmt_errcode = result; + result = ZSTDMT_ERROR(compression_library); + /* fall through */ + error_lock: + pthread_mutex_lock(&ctx->write_mutex); + error_unlock: + list_move(&wl->node, &ctx->writelist_free); + pthread_mutex_unlock(&ctx->write_mutex); + if (in->allocated) + free(in->buf); + return (void *)result; +} + +/* single threaded */ +static size_t st_decompress(void *arg) +{ + ZSTDMT_DCtx *ctx = (ZSTDMT_DCtx *) arg; + cwork_t *w = &ctx->cwork[0]; + ZSTDMT_Buffer In, Out; + ZSTDMT_Buffer *in = &In; + ZSTDMT_Buffer *out = &Out; + ZSTDMT_Buffer *magic = &w->in; + size_t result; + int rv; + + ZSTD_inBuffer zIn; + ZSTD_outBuffer zOut; + + /* init dstream stream */ + result = ZSTD_initDStream(w->dctx); + if (ZSTD_isError(result)) { + zstdmt_errcode = result; + return ZSTDMT_ERROR(compression_library); + } + + /* allocate space for input buffer */ + in->size = ZSTD_DStreamInSize(); + in->buf = malloc(in->size); + if (!in->buf) + return ZSTDMT_ERROR(memory_allocation); + in->allocated = in->size; + + /* allocate space for output buffer */ + out->size = ZSTD_DStreamOutSize(); + out->buf = malloc(out->size); + if (!out->buf) { + free(in->buf); + return ZSTDMT_ERROR(memory_allocation); + } + out->allocated = out->size; + + /* we read already some bytes, handle that: */ + { + /* remember in->buf */ + unsigned char *buf = in->buf; + + /* fill first read bytes to buffer... */ + memcpy(in->buf, magic->buf, magic->size); + magic->buf = in->buf; + in->buf = buf + magic->size; + in->size = in->allocated - magic->size; + + /* read more bytes, to fill buffer */ + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) { + result = ZSTDMT_ERROR(read_fail); + goto error; + } + + /* ready, first buffer complete */ + in->buf = buf; + in->size += magic->size; + ctx->insize += in->size; + } + + zIn.src = in->buf; + zIn.size = in->size; + zIn.pos = 0; + + zOut.dst = out->buf; + + for (;;) { + for (;;) { + /* decompress loop */ + zOut.size = out->allocated; + zOut.pos = 0; + + result = ZSTD_decompressStream(w->dctx, &zOut, &zIn); + if (ZSTD_isError(result)) + goto error_clib; + + if (zOut.pos) { + ZSTDMT_Buffer w; + w.size = zOut.pos; + w.buf = zOut.dst; + rv = ctx->fn_write(ctx->arg_write, &w); + ctx->outsize += zOut.pos; + } + + /* one more round */ + if ((zIn.pos == zIn.size) && (result == 1) && zOut.pos) + continue; + + /* finished */ + if (zIn.pos == zIn.size) + break; + + /* end of frame */ + if (result == 0) { + result = ZSTD_resetDStream(w->dctx); + if (ZSTD_isError(result)) + goto error_clib; + } + } /* decompress */ + + /* read next input */ + in->size = in->allocated; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) { + result = ZSTDMT_ERROR(read_fail); + goto error; + } + + if (in->size == 0) + goto okay; + ctx->insize += in->size; + + zIn.size = in->size; + zIn.pos = 0; + } /* read */ + + error_clib: + zstdmt_errcode = result; + result = ZSTDMT_ERROR(compression_library); + /* fall through */ + error: + /* return with error */ + free(out->buf); + free(in->buf); + return result; + okay: + /* no error */ + free(out->buf); + free(in->buf); + return 0; +} + +#define TYPE_UNKNOWN 0 +#define TYPE_SINGLE_THREAD 1 +#define TYPE_MULTI_THREAD 2 + +size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr) +{ + unsigned char buf[16]; + ZSTDMT_Buffer In; + ZSTDMT_Buffer *in = &In; + cwork_t *w; + int t, rv, type = TYPE_UNKNOWN; + + if (!ctx) + return ZSTDMT_ERROR(compressionParameter_unsupported); + + /* init reading and writing functions */ + ctx->fn_read = rdwr->fn_read; + ctx->fn_write = rdwr->fn_write; + ctx->arg_read = rdwr->arg_read; + ctx->arg_write = rdwr->arg_write; + + /** + * possible valid magic's for us, we need 16 bytes, for checking + * + * 1) ZSTDMT_MAGIC @0 -> ST Stream + * 2) ZSTDMT_MAGIC @0 + MAGIC_SKIPPABLE @9 -> MT Stream else ST + * 3) MAGIC_SKIPPABLE @0 + ZSTDMT_MAGIC @12 -> MT Stream + * 4) all other: not valid! + */ + + /* check for ZSTDMT_MAGIC_SKIPPABLE */ + in->buf = buf; + in->size = 16; + rv = ctx->fn_read(ctx->arg_read, in); + if (rv == -1) + return ZSTDMT_ERROR(read_fail); + + /* must be single threaded standard zstd, when smaller 16 bytes */ + if (in->size < 16) { + if (!IsZstd_Magic(buf)) + return ZSTDMT_ERROR(data_error); + dprintf("single thread style, current pos=%zu\n", in->size); + type = TYPE_SINGLE_THREAD; + if (in->size == 9) { + /* create empty file */ + ctx->threads = 0; + ctx->cwork = 0; + return 0; + } + } else { + if (IsZstd_Skippable(buf) && IsZstd_Magic(buf + 12)) { + /* pzstd */ + dprintf("pzstd style\n"); + type = TYPE_MULTI_THREAD; + } else if (IsZstd_Magic(buf) && IsZstd_Skippable(buf + 9)) { + /* zstdmt */ + dprintf("zstdmt style\n"); + type = TYPE_MULTI_THREAD; + /* set buffer to the */ + } else if (IsZstd_Magic(buf)) { + /* some std zstd stream */ + dprintf("single thread style, current pos=%zu\n", + in->size); + type = TYPE_SINGLE_THREAD; + } else { + /* invalid */ + dprintf("not valid\n"); + return ZSTDMT_ERROR(data_error); + } + } + + /* use single thread extraction, when only one thread is there */ + if (ctx->threadswanted == 1) + type = TYPE_SINGLE_THREAD; + + /* single threaded, but with known sizes */ + if (type == TYPE_SINGLE_THREAD) { + ctx->threads = 1; + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t)); + if (!ctx->cwork) + return ZSTDMT_ERROR(memory_allocation); + w = &ctx->cwork[0]; + w->in.buf = in->buf; + w->in.size = in->size; + w->in.allocated = 0; + w->ctx = ctx; + w->dctx = ZSTD_createDStream(); + if (!w->dctx) + return ZSTDMT_ERROR(memory_allocation); + + /* test, if pt_decompress is better... */ + return st_decompress(ctx); + } + + /* setup thread work */ + ctx->threads = ctx->threadswanted; + ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * ctx->threads); + if (!ctx->cwork) + return ZSTDMT_ERROR(memory_allocation); + + for (t = 0; t < ctx->threads; t++) { + w = &ctx->cwork[t]; + /* one of the threads must reuse the first bytes */ + w->in.buf = in->buf; + w->in.size = in->size; + w->in.allocated = 0; + w->ctx = ctx; + w->dctx = ZSTD_createDStream(); + if (!w->dctx) + return ZSTDMT_ERROR(memory_allocation); + } + + /* real multi threaded, init pthread's */ + pthread_mutex_init(&ctx->read_mutex, NULL); + pthread_mutex_init(&ctx->write_mutex, NULL); + pthread_mutex_init(&ctx->error_mutex, NULL); + + INIT_LIST_HEAD(&ctx->writelist_free); + INIT_LIST_HEAD(&ctx->writelist_busy); + INIT_LIST_HEAD(&ctx->writelist_done); + + /* multi threaded */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + pthread_create(&w->pthread, NULL, pt_decompress, w); + } + + /* wait for all workers */ + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + void *p; + pthread_join(w->pthread, &p); + if (p) + return (size_t) p; + } + + /* clean up pthread stuff */ + pthread_mutex_destroy(&ctx->read_mutex); + pthread_mutex_destroy(&ctx->write_mutex); + pthread_mutex_destroy(&ctx->error_mutex); + + /* clean up the buffers */ + while (!list_empty(&ctx->writelist_free)) { + struct writelist *wl; + struct list_head *entry; + entry = list_first(&ctx->writelist_free); + wl = list_entry(entry, struct writelist, node); + free(wl->out.buf); + list_del(&wl->node); + free(wl); + } + + return 0; +} + +/* returns current uncompressed data size */ +size_t ZSTDMT_GetInsizeDCtx(ZSTDMT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->insize; +} + +/* returns the current compressed data size */ +size_t ZSTDMT_GetOutsizeDCtx(ZSTDMT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->outsize; +} + +/* returns the current compressed frames */ +size_t ZSTDMT_GetFramesDCtx(ZSTDMT_DCtx * ctx) +{ + if (!ctx) + return 0; + + return ctx->curframe; +} + +void ZSTDMT_freeDCtx(ZSTDMT_DCtx * ctx) +{ + int t; + + if (!ctx) + return; + + for (t = 0; t < ctx->threads; t++) { + cwork_t *w = &ctx->cwork[t]; + ZSTD_freeDStream(w->dctx); + } + + if (ctx->cwork) + free(ctx->cwork); + + free(ctx); + ctx = 0; + + return; +} diff --git a/CPP/7zip/7zip.mak b/CPP/7zip/7zip.mak index 71475553..4c74813c 100644 --- a/CPP/7zip/7zip.mak +++ b/CPP/7zip/7zip.mak @@ -27,7 +27,10 @@ OBJS = \ $(COMPRESS_OBJS) \ $(CRYPTO_OBJS) \ $(C_OBJS) \ + $(LZ4_OBJS) \ + $(LZ5_OBJS) \ $(ZSTD_OBJS) \ + $(ZSTDMT_OBJS) \ $(ASM_OBJS) \ $O\resource.res \ @@ -173,11 +176,26 @@ $(C_OBJS): ../../../../C/$(*B).c $(COMPL_O2) !ENDIF +!IFDEF LZ4_OBJS +$(LZ4_OBJS): ../../../../C/lz4/$(*B).c + $(COMPL_O2) +!ENDIF + +!IFDEF LZ5_OBJS +$(LZ5_OBJS): ../../../../C/lz5/$(*B).c + $(COMPL_O2) +!ENDIF + !IFDEF ZSTD_OBJS $(ZSTD_OBJS): ../../../../C/zstd/$(*B).c $(COMPL_O2) !ENDIF +!IFDEF ZSTDMT_OBJS +$(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c + $(COMPL_O2) +!ENDIF + !ELSE @@ -240,8 +258,17 @@ $(ZSTD_OBJS): ../../../../C/zstd/$(*B).c $(COMPLB_O2) {../../../../C}.c{$O}.obj:: $(CCOMPLB) +{../../../../C/lz4}.c{$O}.obj:: + $(CCOMPLB) +{../../../../C/lz5}.c{$O}.obj:: + $(CCOMPLB) {../../../../C/zstd}.c{$O}.obj:: $(CCOMPLB) +{../../../../C/zstdmt}.c{$O}.obj:: + $(CCOMPLB) \ + -I ../../../../C/lz4 \ + -I ../../../../C/lz5 \ + -I ../../../../C/zstd !ENDIF diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index 509a3302..6588d3e3 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -558,7 +558,7 @@ static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param } static const char *g_Exts = - " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo" + " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lz tlz lz4 tlz4 lz5 tlz5 lzh lzo lzx pak rar rpm sit zoo zst" " zip jar ear war msi" " 3gp avi mov mpeg mpg mpe wmv" " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav" diff --git a/CPP/7zip/Archive/IArchive.h b/CPP/7zip/Archive/IArchive.h index 5c0957e5..ed035374 100644 --- a/CPP/7zip/Archive/IArchive.h +++ b/CPP/7zip/Archive/IArchive.h @@ -155,7 +155,7 @@ IArchiveExtractCallback::GetStream() if (IProgress::SetTotal() was called) { IProgress::SetCompleted(completeValue) uses - packSize - for some stream formats (xz, gz, bz2, lzma, z, ppmd). + packSize - for some stream formats (xz, gz, bz2, lz, lzma, z, ppmd). unpackSize - for another formats. } else diff --git a/CPP/7zip/Archive/Icons/lz4.ico b/CPP/7zip/Archive/Icons/lz4.ico new file mode 100644 index 00000000..a2bbb7ed Binary files /dev/null and b/CPP/7zip/Archive/Icons/lz4.ico differ diff --git a/CPP/7zip/Archive/Icons/lz5.ico b/CPP/7zip/Archive/Icons/lz5.ico new file mode 100644 index 00000000..b821c4df Binary files /dev/null and b/CPP/7zip/Archive/Icons/lz5.ico differ diff --git a/CPP/7zip/Archive/Lz4Handler.cpp b/CPP/7zip/Archive/Lz4Handler.cpp new file mode 100644 index 00000000..f709cae2 --- /dev/null +++ b/CPP/7zip/Archive/Lz4Handler.cpp @@ -0,0 +1,366 @@ +// ZstdHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" +#include "../../Common/ComTry.h" +#include "../../Common/Defs.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/Lz4Decoder.h" +#include "../Compress/Lz4Encoder.h" +#include "../Compress/CopyCoder.h" + +#include "Common/DummyOutStream.h" +#include "Common/HandlerOut.h" + +using namespace NWindows; + +namespace NArchive { +namespace NLZ4 { + +class CHandler: + public IInArchive, + public IArchiveOpenSeq, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + CMyComPtr _stream; + CMyComPtr _seqStream; + + bool _isArc; + bool _dataAfterEnd; + bool _needMoreInput; + + bool _packSize_Defined; + bool _unpackSize_Defined; + + UInt64 _packSize; + UInt64 _unpackSize; + UInt64 _numStreams; + UInt64 _numBlocks; + + CSingleMethodProps _props; + +public: + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveOpenSeq, + IOutArchive, + ISetProperties) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); + + CHandler() { } +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams, + kpidNumBlocks +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/) +{ + return S_OK; +} + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; + case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + } + prop.Detach(value); + return S_OK; +} + +static const unsigned kSignatureCheckSize = 4; + +API_FUNC_static_IsArc IsArc_lz4(const Byte *p, size_t size) +{ + if (size < 4) + return k_IsArc_Res_NEED_MORE; + + UInt32 magic = GetUi32(p); + + // skippable frames + if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) { + if (size < 16) + return k_IsArc_Res_NEED_MORE; + magic = GetUi32(p+12); + } + + // lz4 magic + if (magic == 0x184D2204) + return k_IsArc_Res_YES; + + return k_IsArc_Res_NO; +} +} + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *) +{ + COM_TRY_BEGIN + Close(); + { + Byte buf[kSignatureCheckSize]; + RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize)); + if (IsArc_lz4(buf, kSignatureCheckSize) == k_IsArc_Res_NO) + return S_FALSE; + _isArc = true; + _stream = stream; + _seqStream = stream; + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + } + return S_OK; + COM_TRY_END +} + +STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + +STDMETHODIMP CHandler::Close() +{ + _isArc = false; + _dataAfterEnd = false; + _needMoreInput = false; + + _packSize_Defined = false; + _unpackSize_Defined = false; + + _packSize = 0; + + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + + if (_packSize_Defined) + extractCallback->SetTotal(_packSize); + + CMyComPtr realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + Int32 opRes; + + { + + NCompress::NLZ4::CDecoder *decoderSpec = new NCompress::NLZ4::CDecoder; + CMyComPtr decoder = decoderSpec; + decoderSpec->SetInStream(_seqStream); + + CDummyOutStream *outStreamSpec = new CDummyOutStream; + CMyComPtr outStream(outStreamSpec); + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + + realOutStream.Release(); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, true); + + UInt64 packSize = 0; + UInt64 unpackedSize = 0; + + HRESULT result = S_OK; + + for (;;) + { + lps->InSize = packSize; + lps->OutSize = unpackedSize; + + RINOK(lps->SetCur()); + result = decoderSpec->CodeResume(outStream, &unpackedSize, progress); + UInt64 streamSize = decoderSpec->GetInputProcessedSize(); + + if (result != S_FALSE && result != S_OK) + return result; + + if (unpackedSize == 0) + break; + + if (streamSize == packSize) + { + // no new bytes in input stream, So it's good end of archive. + result = S_OK; + break; + } + + if (packSize > streamSize) + return E_FAIL; + + if (result != S_OK) + break; + } + + decoderSpec->ReleaseInStream(); + outStream.Release(); + + if (!_isArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (_needMoreInput) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (_dataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (result == S_FALSE) + opRes = NExtract::NOperationResult::kDataError; + else if (result == S_OK) + opRes = NExtract::NOperationResult::kOK; + else + return result; + + } + + return extractCallback->SetOperationResult(opRes); + + COM_TRY_END +} + +static HRESULT UpdateArchive( + UInt64 unpackSize, + ISequentialOutStream *outStream, + const CProps &props, + IArchiveUpdateCallback *updateCallback) +{ + RINOK(updateCallback->SetTotal(unpackSize)); + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + CLocalProgress *localProgressSpec = new CLocalProgress; + CMyComPtr localProgress = localProgressSpec; + localProgressSpec->Init(updateCallback, true); + NCompress::NLZ4::CEncoder *encoderSpec = new NCompress::NLZ4::CEncoder; + CMyComPtr encoder = encoderSpec; + RINOK(props.SetCoderProps(encoderSpec, NULL)); + RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress)); + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); +} + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *updateCallback) +{ + COM_TRY_BEGIN + + if (numItems != 1) + return E_INVALIDARG; + + Int32 newData, newProps; + UInt32 indexInArchive; + if (!updateCallback) + return E_FAIL; + RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive)); + + if ((newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop)); + if (prop.vt != VT_EMPTY) + if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE) + return E_INVALIDARG; + } + } + + if ((newData)) + { + UInt64 size; + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + } + return UpdateArchive(size, outStream, _props, updateCallback); + } + + if (indexInArchive != 0) + return E_INVALIDARG; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, 0, + NUpdateNotifyOp::kReplicate)) + } + + if (_stream) + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + + return NCompress::CopyStream(_stream, outStream, progress); + + COM_TRY_END +} + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + return _props.SetProperties(names, values, numProps); +} + +static const Byte k_Signature[] = "0x184D2204"; + +REGISTER_ARC_IO( + "lz4", "lz4 tlz4", "* .tar", 0x0f, + k_Signature, + 0, + NArcInfoFlags::kKeepName, + IsArc_lz4) + +}} diff --git a/CPP/7zip/Archive/Lz5Handler.cpp b/CPP/7zip/Archive/Lz5Handler.cpp new file mode 100644 index 00000000..49485ee1 --- /dev/null +++ b/CPP/7zip/Archive/Lz5Handler.cpp @@ -0,0 +1,366 @@ +// ZstdHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" +#include "../../Common/ComTry.h" +#include "../../Common/Defs.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/Lz5Decoder.h" +#include "../Compress/Lz5Encoder.h" +#include "../Compress/CopyCoder.h" + +#include "Common/DummyOutStream.h" +#include "Common/HandlerOut.h" + +using namespace NWindows; + +namespace NArchive { +namespace NLZ5 { + +class CHandler: + public IInArchive, + public IArchiveOpenSeq, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + CMyComPtr _stream; + CMyComPtr _seqStream; + + bool _isArc; + bool _dataAfterEnd; + bool _needMoreInput; + + bool _packSize_Defined; + bool _unpackSize_Defined; + + UInt64 _packSize; + UInt64 _unpackSize; + UInt64 _numStreams; + UInt64 _numBlocks; + + CSingleMethodProps _props; + +public: + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveOpenSeq, + IOutArchive, + ISetProperties) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); + + CHandler() { } +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams, + kpidNumBlocks +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/) +{ + return S_OK; +} + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; + case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + } + prop.Detach(value); + return S_OK; +} + +static const unsigned kSignatureCheckSize = 4; + +API_FUNC_static_IsArc IsArc_lz5(const Byte *p, size_t size) +{ + if (size < 4) + return k_IsArc_Res_NEED_MORE; + + UInt32 magic = GetUi32(p); + + // skippable frames + if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) { + if (size < 16) + return k_IsArc_Res_NEED_MORE; + magic = GetUi32(p+12); + } + + // Lz5 Magic + if (magic == 0x184D2205) + return k_IsArc_Res_YES; + + return k_IsArc_Res_NO; +} +} + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *) +{ + COM_TRY_BEGIN + Close(); + { + Byte buf[kSignatureCheckSize]; + RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize)); + if (IsArc_lz5(buf, kSignatureCheckSize) == k_IsArc_Res_NO) + return S_FALSE; + _isArc = true; + _stream = stream; + _seqStream = stream; + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + } + return S_OK; + COM_TRY_END +} + +STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + +STDMETHODIMP CHandler::Close() +{ + _isArc = false; + _dataAfterEnd = false; + _needMoreInput = false; + + _packSize_Defined = false; + _unpackSize_Defined = false; + + _packSize = 0; + + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + + if (_packSize_Defined) + extractCallback->SetTotal(_packSize); + + CMyComPtr realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + Int32 opRes; + + { + + NCompress::NLZ5::CDecoder *decoderSpec = new NCompress::NLZ5::CDecoder; + CMyComPtr decoder = decoderSpec; + decoderSpec->SetInStream(_seqStream); + + CDummyOutStream *outStreamSpec = new CDummyOutStream; + CMyComPtr outStream(outStreamSpec); + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + + realOutStream.Release(); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, true); + + UInt64 packSize = 0; + UInt64 unpackedSize = 0; + + HRESULT result = S_OK; + + for (;;) + { + lps->InSize = packSize; + lps->OutSize = unpackedSize; + + RINOK(lps->SetCur()); + result = decoderSpec->CodeResume(outStream, &unpackedSize, progress); + UInt64 streamSize = decoderSpec->GetInputProcessedSize(); + + if (result != S_FALSE && result != S_OK) + return result; + + if (unpackedSize == 0) + break; + + if (streamSize == packSize) + { + // no new bytes in input stream, So it's good end of archive. + result = S_OK; + break; + } + + if (packSize > streamSize) + return E_FAIL; + + if (result != S_OK) + break; + } + + decoderSpec->ReleaseInStream(); + outStream.Release(); + + if (!_isArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (_needMoreInput) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (_dataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (result == S_FALSE) + opRes = NExtract::NOperationResult::kDataError; + else if (result == S_OK) + opRes = NExtract::NOperationResult::kOK; + else + return result; + + } + + return extractCallback->SetOperationResult(opRes); + + COM_TRY_END +} + +static HRESULT UpdateArchive( + UInt64 unpackSize, + ISequentialOutStream *outStream, + const CProps &props, + IArchiveUpdateCallback *updateCallback) +{ + RINOK(updateCallback->SetTotal(unpackSize)); + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + CLocalProgress *localProgressSpec = new CLocalProgress; + CMyComPtr localProgress = localProgressSpec; + localProgressSpec->Init(updateCallback, true); + NCompress::NLZ5::CEncoder *encoderSpec = new NCompress::NLZ5::CEncoder; + CMyComPtr encoder = encoderSpec; + RINOK(props.SetCoderProps(encoderSpec, NULL)); + RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress)); + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); +} + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *updateCallback) +{ + COM_TRY_BEGIN + + if (numItems != 1) + return E_INVALIDARG; + + Int32 newData, newProps; + UInt32 indexInArchive; + if (!updateCallback) + return E_FAIL; + RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive)); + + if ((newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop)); + if (prop.vt != VT_EMPTY) + if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE) + return E_INVALIDARG; + } + } + + if ((newData)) + { + UInt64 size; + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + } + return UpdateArchive(size, outStream, _props, updateCallback); + } + + if (indexInArchive != 0) + return E_INVALIDARG; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, 0, + NUpdateNotifyOp::kReplicate)) + } + + if (_stream) + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + + return NCompress::CopyStream(_stream, outStream, progress); + + COM_TRY_END +} + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + return _props.SetProperties(names, values, numProps); +} + +static const Byte k_Signature[] = "0x184D2205"; + +REGISTER_ARC_IO( + "lz5", "lz5 tlz5", "* .tar", 0x10, + k_Signature, + 0, + NArcInfoFlags::kKeepName, + IsArc_lz5) + +}} diff --git a/CPP/7zip/Archive/LzHandler.cpp b/CPP/7zip/Archive/LzHandler.cpp new file mode 100644 index 00000000..dd2e4c53 --- /dev/null +++ b/CPP/7zip/Archive/LzHandler.cpp @@ -0,0 +1,460 @@ +// LzHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" + +#include "../../Common/ComTry.h" +#include "../../Common/IntToString.h" + +#include "../../Windows/PropVariant.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/LzmaDecoder.h" + +#include "Common/OutStreamWithCRC.h" + +using namespace NWindows; + +namespace NArchive { +namespace NLz { + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize, +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams +}; + +static const Byte k_Signature[5] = { 'L', 'Z', 'I', 'P', 1 }; +enum { k_SignatureSize = 5 }; + +struct CHeader +{ + Byte data[6]; // 0-3 magic bytes, 4 version, 5 coded_dict_size + enum { size = 6 }; + enum { min_dictionary_size = 1 << 12, max_dictionary_size = 1 << 29 }; + unsigned DicSize; + Byte LzmaProps[5]; + + bool Parse(); +}; + +struct CTrailer +{ + Byte data[20]; // 0-3 CRC32 of the uncompressed data + // 4-11 size of the uncompressed data + // 12-19 member size including header and trailer + enum { size = 20 }; + + unsigned data_crc() const + { + unsigned tmp = 0; + for( int i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + + UInt64 data_size() const + { + UInt64 tmp = 0; + for( int i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + + UInt64 member_size() const + { + UInt64 tmp = 0; + for( int i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } +}; + +class CDecoder +{ + CMyComPtr _lzmaDecoder; +public: + NCompress::NLzma::CDecoder *_lzmaDecoderSpec; + + ~CDecoder(); + HRESULT Create(ISequentialInStream *inStream); + + HRESULT Code(const CHeader &header, ISequentialOutStream *outStream, ICompressProgressInfo *progress); + + UInt64 GetInputProcessedSize() const { return _lzmaDecoderSpec->GetInputProcessedSize(); } + + void ReleaseInStream() { if (_lzmaDecoder) _lzmaDecoderSpec->ReleaseInStream(); } + + HRESULT ReadInput(Byte *data, UInt32 size, UInt32 *processedSize) + { return _lzmaDecoderSpec->ReadFromInputStream(data, size, processedSize); } +}; + +HRESULT CDecoder::Create(ISequentialInStream *inStream) +{ + if (!_lzmaDecoder) + { + _lzmaDecoderSpec = new NCompress::NLzma::CDecoder; + _lzmaDecoderSpec->FinishStream = true; + _lzmaDecoder = _lzmaDecoderSpec; + } + + return _lzmaDecoderSpec->SetInStream(inStream); +} + +CDecoder::~CDecoder() +{ + ReleaseInStream(); +} + +HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, + ICompressProgressInfo *progress) +{ + { + CMyComPtr setDecoderProperties; + _lzmaDecoder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecoderProperties); + if (!setDecoderProperties) + return E_NOTIMPL; + RINOK(setDecoderProperties->SetDecoderProperties2(header.LzmaProps, 5)); + } + + RINOK(_lzmaDecoderSpec->CodeResume(outStream, NULL, progress)); + + return S_OK; +} + + +class CHandler: + public IInArchive, + public IArchiveOpenSeq, + public CMyUnknownImp +{ + CHeader _header; + CMyComPtr _stream; + CMyComPtr _seqStream; + + bool _isArc; + bool _needSeekToStart; + bool _dataAfterEnd; + bool _needMoreInput; + + bool _packSize_Defined; + bool _unpackSize_Defined; + bool _numStreams_Defined; + + bool _dataError; + + UInt64 _packSize; + UInt64 _unpackSize; + UInt64 _numStreams; + +public: + MY_UNKNOWN_IMP2(IInArchive, IArchiveOpenSeq) + + INTERFACE_IInArchive(;) + STDMETHOD(OpenSeq)(ISequentialInStream *stream); + + CHandler() { } +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPhySize: if (_packSize_Defined) prop = _packSize; break; + case kpidUnpackSize: if (_unpackSize_Defined) prop = _unpackSize; break; + case kpidNumStreams: if (_numStreams_Defined) prop = _numStreams; break; + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;; + if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; + if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd; + if (_dataError) v |= kpv_ErrorFlags_DataError; + prop = v; + } + } + prop.Detach(value); + return S_OK; +} + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; + } + prop.Detach(value); + return S_OK; +} + +API_FUNC_static_IsArc IsArc_Lz(const Byte *p, size_t size) +{ + if (size < k_SignatureSize) + return k_IsArc_Res_NEED_MORE; + for( int i = 0; i < k_SignatureSize; ++i ) + if( p[i] != k_Signature[i] ) + return k_IsArc_Res_NO; + return k_IsArc_Res_YES; +} +} + +bool CHeader::Parse() +{ + if (IsArc_Lz(data, k_SignatureSize) == k_IsArc_Res_NO) + return false; + DicSize = ( 1 << ( data[5] & 0x1F ) ); + if( DicSize > min_dictionary_size ) + DicSize -= ( DicSize / 16 ) * ( ( data[5] >> 5 ) & 7 ); + LzmaProps[0] = 93; /* (45 * 2) + (9 * 0) + 3 */ + unsigned ds = DicSize; + for( int i = 1; i <= 4; ++i ) { LzmaProps[i] = ds & 0xFF; ds >>= 8; } + return (DicSize >= min_dictionary_size && DicSize <= max_dictionary_size); +} + +STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *) +{ + Close(); + + RINOK(ReadStream_FALSE(inStream, _header.data, CHeader::size)); + + if (!_header.Parse()) + return S_FALSE; + + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_packSize)); + if (_packSize < 36) + return S_FALSE; + _isArc = true; + _stream = inStream; + _seqStream = inStream; + _needSeekToStart = true; + return S_OK; +} + +STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + +STDMETHODIMP CHandler::Close() +{ + _isArc = false; + _packSize_Defined = false; + _unpackSize_Defined = false; + _numStreams_Defined = false; + + _dataAfterEnd = false; + _needMoreInput = false; + _dataError = false; + + _packSize = 0; + + _needSeekToStart = false; + + _stream.Release(); + _seqStream.Release(); + return S_OK; +} + +class CCompressProgressInfoImp: + public ICompressProgressInfo, + public CMyUnknownImp +{ + CMyComPtr Callback; +public: + UInt64 Offset; + + MY_UNKNOWN_IMP1(ICompressProgressInfo) + STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); + void Init(IArchiveOpenCallback *callback) { Callback = callback; } +}; + +STDMETHODIMP CCompressProgressInfoImp::SetRatioInfo(const UInt64 *inSize, const UInt64 * /* outSize */) +{ + if (Callback) + { + UInt64 files = 0; + UInt64 value = Offset + *inSize; + return Callback->SetCompleted(&files, &value); + } + return S_OK; +} + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + + if (_packSize_Defined) + extractCallback->SetTotal(_packSize); + + CMyComPtr realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC; + CMyComPtr outStream(outStreamSpec); + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + realOutStream.Release(); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, true); + + if (_needSeekToStart) + { + if (!_stream) + return E_FAIL; + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + } + else + _needSeekToStart = true; + + CDecoder decoder; + HRESULT result = decoder.Create(_seqStream); + RINOK(result); + + bool firstItem = true; + + UInt64 packSize = 0; + UInt64 unpackSize = 0; + UInt64 numStreams = 0; + + bool crcError = false; + bool dataAfterEnd = false; + + for (;;) + { + lps->InSize = packSize; + lps->OutSize = unpackSize; + RINOK(lps->SetCur()); + + CHeader st; + UInt32 processed; + RINOK(decoder.ReadInput(st.data, CHeader::size, &processed)); + if (processed != CHeader::size) + { + if (processed != 0) + dataAfterEnd = true; + break; + } + + if (!st.Parse()) + { + dataAfterEnd = true; + break; + } + numStreams++; + firstItem = false; + outStreamSpec->InitCRC(); + + result = decoder.Code(st, outStream, progress); + + UInt64 member_size = decoder.GetInputProcessedSize() - packSize; + packSize += member_size; + UInt64 data_size = outStreamSpec->GetSize() - unpackSize; + unpackSize += data_size; + + if (result == S_OK) + { + CTrailer trailer; + RINOK(decoder.ReadInput(trailer.data, CTrailer::size, &processed)); + packSize += processed; member_size += processed; + if (processed != CTrailer::size || + trailer.data_crc() != outStreamSpec->GetCRC() || + trailer.data_size() != data_size || + trailer.member_size() != member_size) + { + crcError = true; + result = S_FALSE; + break; + } + } + if (result == S_FALSE) + break; + RINOK(result); + } + + if (firstItem) + { + _isArc = false; + result = S_FALSE; + } + else if (result == S_OK || result == S_FALSE) + { + if (dataAfterEnd) + _dataAfterEnd = true; + else if (decoder._lzmaDecoderSpec->NeedMoreInput) + _needMoreInput = true; + + _packSize = packSize; + _unpackSize = unpackSize; + _numStreams = numStreams; + + _packSize_Defined = true; + _unpackSize_Defined = true; + _numStreams_Defined = true; + } + + Int32 opResult = NExtract::NOperationResult::kOK; + + if (!_isArc) + opResult = NExtract::NOperationResult::kIsNotArc; + else if (_needMoreInput) + opResult = NExtract::NOperationResult::kUnexpectedEnd; + else if (crcError) + opResult = NExtract::NOperationResult::kCRCError; + else if (_dataAfterEnd) + opResult = NExtract::NOperationResult::kDataAfterEnd; + else if (result == S_FALSE) + opResult = NExtract::NOperationResult::kDataError; + else if (result == S_OK) + opResult = NExtract::NOperationResult::kOK; + else + return result; + + outStream.Release(); + return extractCallback->SetOperationResult(opResult); + + COM_TRY_END +} + +REGISTER_ARC_I( + "lzip", "lz tlz", "* .tar", 0xC6, + k_Signature, + 0, + NArcInfoFlags::kKeepName, + IsArc_Lz) + +}} diff --git a/CPP/7zip/Archive/ZstdHandler.cpp b/CPP/7zip/Archive/ZstdHandler.cpp index 32ac99b1..066bb8dd 100644 --- a/CPP/7zip/Archive/ZstdHandler.cpp +++ b/CPP/7zip/Archive/ZstdHandler.cpp @@ -17,8 +17,6 @@ #include "Common/DummyOutStream.h" #include "Common/HandlerOut.h" -#include - using namespace NWindows; namespace NArchive { @@ -109,6 +107,14 @@ API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size) UInt32 magic = GetUi32(p); + // skippable frames + if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) { + if (size < 16) + return k_IsArc_Res_NEED_MORE; + magic = GetUi32(p+12); + } + +#ifdef ZSTD_LEGACY_SUPPORT // zstd 0.1 if (magic == 0xFD2FB51E) return k_IsArc_Res_YES; @@ -116,10 +122,11 @@ API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size) // zstd magic's for 0.2 .. 0.8 (aka 1.x) if (magic >= 0xFD2FB522 && magic <= 0xFD2FB528) return k_IsArc_Res_YES; - - // skippable frames - if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) +#else + /* only version 1.x */ + if (magic == 0xFD2FB528) return k_IsArc_Res_YES; +#endif return k_IsArc_Res_NO; } @@ -223,10 +230,6 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, result = decoderSpec->CodeResume(outStream, &unpackedSize, progress); UInt64 streamSize = decoderSpec->GetInputProcessedSize(); - printf("streamsize=%d packsize=%d unpackedSize=%d\n", - streamSize, packSize, unpackedSize); - fflush(stdout); - if (result != S_FALSE && result != S_OK) return result; @@ -362,7 +365,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR return _props.SetProperties(names, values, numProps); } -static const Byte k_Signature[] = "0xFD2FB525..0xFD2FB528"; +static const Byte k_Signature[] = "0xFD2FB522..28"; REGISTER_ARC_IO( "zstd", "zst tzstd", "* .tar", 0x0e, diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index 82f5acf4..addbcaae 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -1618,6 +1618,10 @@ SOURCE=..\..\Archive\IArchive.h # End Source File # Begin Source File +SOURCE=..\..\Archive\LzHandler.cpp +# End Source File +# Begin Source File + SOURCE=..\..\Archive\LzmaHandler.cpp # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index a91b2892..504cdd62 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -70,6 +70,7 @@ AR_OBJS = \ $O\Bz2Handler.obj \ $O\DeflateProps.obj \ $O\GzHandler.obj \ + $O\LzHandler.obj \ $O\LzmaHandler.obj \ $O\SplitHandler.obj \ $O\XzHandler.obj \ diff --git a/CPP/7zip/Bundles/Codec_lz4/StdAfx.cpp b/CPP/7zip/Bundles/Codec_lz4/StdAfx.cpp new file mode 100644 index 00000000..c6d3b1fa --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz4/StdAfx.cpp @@ -0,0 +1,3 @@ +// StdAfx.cpp + +#include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Codec_lz4/StdAfx.h b/CPP/7zip/Bundles/Codec_lz4/StdAfx.h new file mode 100644 index 00000000..59d9ac15 --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz4/StdAfx.h @@ -0,0 +1,8 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../../Common/Common.h" + +#endif diff --git a/CPP/7zip/Bundles/Codec_lz4/makefile b/CPP/7zip/Bundles/Codec_lz4/makefile new file mode 100644 index 00000000..b4846e6c --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz4/makefile @@ -0,0 +1,36 @@ +PROG = lz4.dll +DEF_FILE = ../../Compress/Codec.def +CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID + +7ZIP_COMMON_OBJS = \ + $O\StreamUtils.obj \ + +WIN_OBJS = \ + $O\System.obj \ + +COMPRESS_OBJS = \ + $O\CodecExports.obj \ + $O\DllExportsCompress.obj \ + +C_OBJS = \ + $O\Alloc.obj \ + $O\Threads.obj \ + +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\Lz4Decoder.obj \ + $O\Lz4Encoder.obj \ + $O\Lz4Register.obj \ + +LZ4_OBJS = \ + $O\lz4.obj \ + $O\lz4frame.obj \ + $O\lz4hc.obj \ + $O\lz4xxhash.obj \ + +ZSTDMT_OBJS = \ + $O\lz4mt_common.obj \ + $O\lz4mt_compress.obj \ + $O\lz4mt_decompress.obj \ + $O\threading.obj \ + +!include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Codec_lz4/resource.rc b/CPP/7zip/Bundles/Codec_lz4/resource.rc new file mode 100644 index 00000000..0665b381 --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz4/resource.rc @@ -0,0 +1,6 @@ +#include "../../../../C/7zVersionTr.h" +#include "../../../../C/7zVersion.rc" + +MY_VERSION_INFO_DLL("7z LZ4 Plugin", "lz4") + +101 ICON "../../Archive/Icons/7z.ico" diff --git a/CPP/7zip/Bundles/Codec_lz5/StdAfx.cpp b/CPP/7zip/Bundles/Codec_lz5/StdAfx.cpp new file mode 100644 index 00000000..c6d3b1fa --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz5/StdAfx.cpp @@ -0,0 +1,3 @@ +// StdAfx.cpp + +#include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Codec_lz5/StdAfx.h b/CPP/7zip/Bundles/Codec_lz5/StdAfx.h new file mode 100644 index 00000000..59d9ac15 --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz5/StdAfx.h @@ -0,0 +1,8 @@ +// StdAfx.h + +#ifndef __STDAFX_H +#define __STDAFX_H + +#include "../../../Common/Common.h" + +#endif diff --git a/CPP/7zip/Bundles/Codec_lz5/makefile b/CPP/7zip/Bundles/Codec_lz5/makefile new file mode 100644 index 00000000..a6fcef44 --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz5/makefile @@ -0,0 +1,36 @@ +PROG = lz5.dll +DEF_FILE = ../../Compress/Codec.def +CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID + +7ZIP_COMMON_OBJS = \ + $O\StreamUtils.obj \ + +WIN_OBJS = \ + $O\System.obj \ + +COMPRESS_OBJS = \ + $O\CodecExports.obj \ + $O\DllExportsCompress.obj \ + +C_OBJS = \ + $O\Alloc.obj \ + $O\Threads.obj \ + +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\Lz5Decoder.obj \ + $O\Lz5Encoder.obj \ + $O\Lz5Register.obj \ + +LZ5_OBJS = \ + $O\lz5.obj \ + $O\lz5frame.obj \ + $O\lz5hc.obj \ + $O\lz5xxhash.obj \ + +ZSTDMT_OBJS = \ + $O\lz5mt_common.obj \ + $O\lz5mt_compress.obj \ + $O\lz5mt_decompress.obj \ + $O\threading.obj \ + +!include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Codec_lz5/resource.rc b/CPP/7zip/Bundles/Codec_lz5/resource.rc new file mode 100644 index 00000000..2d1d7904 --- /dev/null +++ b/CPP/7zip/Bundles/Codec_lz5/resource.rc @@ -0,0 +1,6 @@ +#include "../../../../C/7zVersionTr.h" +#include "../../../../C/7zVersion.rc" + +MY_VERSION_INFO_DLL("7z LZ5 Plugin", "lz5") + +101 ICON "../../Archive/Icons/7z.ico" diff --git a/CPP/7zip/Bundles/Codec_zstd/makefile b/CPP/7zip/Bundles/Codec_zstd/makefile index 49bd46dc..10c86154 100644 --- a/CPP/7zip/Bundles/Codec_zstd/makefile +++ b/CPP/7zip/Bundles/Codec_zstd/makefile @@ -5,17 +5,38 @@ CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ +WIN_OBJS = \ + $O\System.obj \ + COMPRESS_OBJS = \ $O\CodecExports.obj \ $O\DllExportsCompress.obj \ C_OBJS = \ $O\Alloc.obj \ + $O\Threads.obj \ -!include "../../zstd.mak" -ZSTD_OBJS = $(ZSTD_OBJS) \ +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\ZstdDecoder.obj \ + $O\ZstdEncoder.obj \ + $O\ZstdRegister.obj \ + +ZSTD_OBJS = \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\huf_decompress.obj \ + $O\zstd_common.obj \ + $O\zstd_decompress.obj \ + $O\xxhash.obj \ $O\fse_compress.obj \ $O\huf_compress.obj \ $O\zstd_compress.obj \ + $O\error_private.obj \ + +ZSTDMT_OBJS = \ + $O\threading.obj \ + $O\zstdmt_common.obj \ + $O\zstdmt_compress.obj \ + $O\zstdmt_decompress.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Codec_zstdF/makefile b/CPP/7zip/Bundles/Codec_zstdF/makefile index e3aa9667..b83db520 100644 --- a/CPP/7zip/Bundles/Codec_zstdF/makefile +++ b/CPP/7zip/Bundles/Codec_zstdF/makefile @@ -5,18 +5,33 @@ CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_LEGACY_SUPPORT 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ +WIN_OBJS = \ + $O\System.obj \ + COMPRESS_OBJS = \ $O\CodecExports.obj \ $O\DllExportsCompress.obj \ C_OBJS = \ $O\Alloc.obj \ + $O\Threads.obj \ -!include "../../zstd.mak" -ZSTD_OBJS = $(ZSTD_OBJS) \ +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\ZstdDecoder.obj \ + $O\ZstdEncoder.obj \ + $O\ZstdRegister.obj \ + +ZSTD_OBJS = \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\huf_decompress.obj \ + $O\zstd_common.obj \ + $O\zstd_decompress.obj \ + $O\xxhash.obj \ $O\fse_compress.obj \ $O\huf_compress.obj \ $O\zstd_compress.obj \ + $O\error_private.obj \ $O\zstd_v01.obj \ $O\zstd_v02.obj \ $O\zstd_v03.obj \ @@ -25,4 +40,10 @@ ZSTD_OBJS = $(ZSTD_OBJS) \ $O\zstd_v06.obj \ $O\zstd_v07.obj \ +ZSTDMT_OBJS = \ + $O\threading.obj \ + $O\zstdmt_common.obj \ + $O\zstdmt_compress.obj \ + $O\zstdmt_decompress.obj \ + !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Fm/resource.rc b/CPP/7zip/Bundles/Fm/resource.rc index b3628c40..eddbb6f8 100644 --- a/CPP/7zip/Bundles/Fm/resource.rc +++ b/CPP/7zip/Bundles/Fm/resource.rc @@ -3,5 +3,5 @@ STRINGTABLE BEGIN - 100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs" + 100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lz tlz lz4 tlz4 lz5 tlz5 lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs zst" END diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 8acf8471..ef3b82c5 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -136,10 +136,28 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" -!include "../../zstd.mak" -ZSTD_OBJS = $(ZSTD_OBJS) \ + +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\ZstdDecoder.obj \ + $O\ZstdEncoder.obj \ + $O\ZstdRegister.obj \ + +ZSTD_OBJS = \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\huf_decompress.obj \ + $O\zstd_common.obj \ + $O\zstd_decompress.obj \ + $O\xxhash.obj \ $O\fse_compress.obj \ $O\huf_compress.obj \ $O\zstd_compress.obj \ + $O\error_private.obj \ + +ZSTDMT_OBJS = \ + $O\threading.obj \ + $O\zstdmt_common.obj \ + $O\zstdmt_compress.obj \ + $O\zstdmt_decompress.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index 75ab01a5..30478509 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -69,6 +69,9 @@ AR_OBJS = \ $O\HandlerCont.obj \ $O\HfsHandler.obj \ $O\IhexHandler.obj \ + $O\LzHandler.obj \ + $O\Lz4Handler.obj \ + $O\Lz5Handler.obj \ $O\LzhHandler.obj \ $O\LzmaHandler.obj \ $O\MachoHandler.obj \ @@ -104,7 +107,6 @@ AR_COMMON_OBJS = \ $O\HandlerOut.obj \ $O\ParseProperties.obj \ - 7Z_OBJS = \ $O\7zCompressionMode.obj \ $O\7zDecode.obj \ @@ -242,7 +244,6 @@ CRYPTO_OBJS = \ $O\ZipCrypto.obj \ $O\ZipStrong.obj \ - C_OBJS = \ $O\7zBuf2.obj \ $O\7zStream.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/makefile b/CPP/7zip/Bundles/Format7zF/makefile index 7d0ad987..c4120f17 100644 --- a/CPP/7zip/Bundles/Format7zF/makefile +++ b/CPP/7zip/Bundles/Format7zF/makefile @@ -16,12 +16,39 @@ AR_OBJS = $(AR_OBJS) \ $O\ArchiveExports.obj \ $O\DllExports2.obj \ +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\Lz4Decoder.obj \ + $O\Lz4Encoder.obj \ + $O\Lz4Register.obj \ + $O\Lz5Decoder.obj \ + $O\Lz5Encoder.obj \ + $O\Lz5Register.obj \ + $O\ZstdDecoder.obj \ + $O\ZstdEncoder.obj \ + $O\ZstdRegister.obj \ -!include "../../zstd.mak" -ZSTD_OBJS = $(ZSTD_OBJS) \ +LZ4_OBJS = \ + $O\lz4.obj \ + $O\lz4frame.obj \ + $O\lz4hc.obj \ + $O\lz4xxhash.obj \ + +LZ5_OBJS = \ + $O\lz5.obj \ + $O\lz5frame.obj \ + $O\lz5hc.obj \ + +ZSTD_OBJS = \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\huf_decompress.obj \ + $O\zstd_common.obj \ + $O\zstd_decompress.obj \ + $O\xxhash.obj \ $O\fse_compress.obj \ $O\huf_compress.obj \ $O\zstd_compress.obj \ + $O\error_private.obj \ $O\zstd_v01.obj \ $O\zstd_v02.obj \ $O\zstd_v03.obj \ @@ -30,4 +57,16 @@ ZSTD_OBJS = $(ZSTD_OBJS) \ $O\zstd_v06.obj \ $O\zstd_v07.obj \ +ZSTDMT_OBJS = \ + $O\lz5mt_common.obj \ + $O\lz5mt_compress.obj \ + $O\lz5mt_decompress.obj \ + $O\lz4mt_common.obj \ + $O\lz4mt_compress.obj \ + $O\lz4mt_decompress.obj \ + $O\threading.obj \ + $O\zstdmt_common.obj \ + $O\zstdmt_compress.obj \ + $O\zstdmt_decompress.obj \ + !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zF/resource.rc b/CPP/7zip/Bundles/Format7zF/resource.rc index c38d3542..53d024a8 100644 --- a/CPP/7zip/Bundles/Format7zF/resource.rc +++ b/CPP/7zip/Bundles/Format7zF/resource.rc @@ -29,9 +29,11 @@ MY_VERSION_INFO_DLL("7z Plugin", "7z") 23 ICON "../../Archive/Icons/xz.ico" 24 ICON "../../Archive/Icons/squashfs.ico" 25 ICON "../../Archive/Icons/zst.ico" +26 ICON "../../Archive/Icons/lz4.ico" +27 ICON "../../Archive/Icons/lz5.ico" STRINGTABLE BEGIN - 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25" + 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lz:16 tlz:16 lz4:26 lz5:27 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25" END diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index fa138620..2b565dd2 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -41,6 +41,7 @@ WIN_OBJS = \ $O\PropVariant.obj \ $O\PropVariantConv.obj \ $O\Synchronization.obj \ + $O\System.obj \ 7ZIP_COMMON_OBJS = \ $O\CreateCoder.obj \ @@ -126,5 +127,22 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" -!include "../../zstd.mak" +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\ZstdDecoder.obj \ + $O\ZstdRegister.obj \ + +ZSTD_OBJS = \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\huf_decompress.obj \ + $O\zstd_common.obj \ + $O\zstd_decompress.obj \ + $O\xxhash.obj \ + $O\error_private.obj \ + +ZSTDMT_OBJS = \ + $O\threading.obj \ + $O\zstdmt_common.obj \ + $O\zstdmt_decompress.obj \ + !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXSetup/makefile b/CPP/7zip/Bundles/SFXSetup/makefile index 3b8225a6..2e6eed00 100644 --- a/CPP/7zip/Bundles/SFXSetup/makefile +++ b/CPP/7zip/Bundles/SFXSetup/makefile @@ -106,6 +106,4 @@ C_OBJS = \ $O\Threads.obj \ !include "../../Crc.mak" - -!include "../../zstd.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index 758d65f4..ce088b22 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -40,6 +40,7 @@ WIN_OBJS = \ $O\PropVariantConv.obj \ $O\ResourceString.obj \ $O\Shell.obj \ + $O\System.obj \ $O\Synchronization.obj \ $O\Window.obj \ @@ -144,5 +145,22 @@ C_OBJS = \ !include "../../Aes.mak" !include "../../Crc.mak" -!include "../../zstd.mak" +COMPRESS_OBJS = $(COMPRESS_OBJS) \ + $O\ZstdDecoder.obj \ + $O\ZstdRegister.obj \ + +ZSTD_OBJS = \ + $O\entropy_common.obj \ + $O\fse_decompress.obj \ + $O\huf_decompress.obj \ + $O\zstd_common.obj \ + $O\zstd_decompress.obj \ + $O\xxhash.obj \ + $O\error_private.obj \ + +ZSTDMT_OBJS = \ + $O\threading.obj \ + $O\zstdmt_common.obj \ + $O\zstdmt_decompress.obj \ + !include "../../7zip.mak" diff --git a/CPP/7zip/Compress/Lz4Decoder.cpp b/CPP/7zip/Compress/Lz4Decoder.cpp new file mode 100644 index 00000000..85eb2a55 --- /dev/null +++ b/CPP/7zip/Compress/Lz4Decoder.cpp @@ -0,0 +1,177 @@ +// (C) 2016 Tino Reichardt + +#include "StdAfx.h" +#include "Lz4Decoder.h" + +int Lz4Read(void *arg, LZ4MT_Buffer * in) +{ + struct Lz4Stream *x = (struct Lz4Stream*)arg; + size_t size = in->size; + + HRESULT res = ReadStream(x->inStream, in->buf, &size); + if (res != S_OK) + return -1; + + in->size = size; + *x->processedIn += size; + + return 0; +} + +int Lz4Write(void *arg, LZ4MT_Buffer * out) +{ + struct Lz4Stream *x = (struct Lz4Stream*)arg; + UInt32 todo = (UInt32)out->size; + UInt32 done = 0; + + while (todo != 0) + { + UInt32 block; + HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); + done += block; + if (res == k_My_HRESULT_WritingWasCut) + break; + if (res != S_OK) + return -1; + if (block == 0) + return E_FAIL; + todo -= block; + } + + *x->processedOut += done; + if (x->progress) + x->progress->SetRatioInfo(x->processedIn, x->processedOut); + + return 0; +} + +namespace NCompress { +namespace NLZ4 { + +CDecoder::CDecoder(): + _processedIn(0), + _processedOut(0), + _inputSize(0), + _numThreads(NWindows::NSystem::GetNumberOfProcessors()) +{ + _props.clear(); +} + +CDecoder::~CDecoder() +{ +} + +HRESULT CDecoder::ErrorOut(size_t code) +{ + const char *strError = LZ4MT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + +STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) +{ + DProps *pProps = (DProps *)prop; + + if (size != sizeof(DProps)) + return E_FAIL; + + memcpy(&_props, pProps, sizeof (DProps)); + + return S_OK; +} + +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + const UInt32 kNumThreadsMax = LZ4MT_THREAD_MAX; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + _numThreads = numThreads; + return S_OK; +} + +HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/) +{ + _processedOut = 0; + return S_OK; +} + +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize) +{ + _processedIn = 0; + RINOK(SetOutStreamSizeResume(outSize)); + return S_OK; +} + +HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, + ISequentialOutStream * outStream, ICompressProgressInfo * progress) +{ + LZ4MT_RdWr_t rdwr; + size_t result; + HRESULT res = S_OK; + + struct Lz4Stream Rd; + Rd.inStream = inStream; + Rd.processedIn = &_processedIn; + + struct Lz4Stream Wr; + Wr.progress = progress; + Wr.outStream = outStream; + Wr.processedIn = &_processedIn; + Wr.processedOut = &_processedOut; + + /* 1) setup read/write functions */ + rdwr.fn_read = ::Lz4Read; + rdwr.fn_write = ::Lz4Write; + rdwr.arg_read = (void *)&Rd; + rdwr.arg_write = (void *)&Wr; + + /* 2) create compression context */ + LZ4MT_DCtx *ctx = LZ4MT_createDCtx(_numThreads, _inputSize); + if (!ctx) + return S_FALSE; + + /* 3) compress */ + result = LZ4MT_DecompressDCtx(ctx, &rdwr); + if (result == (size_t)-LZ4MT_error_read_fail) + res = E_ABORT; + else if (LZ4MT_isError(result)) + return ErrorOut(result); + + /* 4) free resources */ + LZ4MT_freeDCtx(ctx); + return res; +} + +STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream, + const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress) +{ + SetOutStreamSize(outSize); + return CodeSpec(inStream, outStream, progress); +} + +#ifndef NO_READ_FROM_CODER +STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream) +{ + _inStream = inStream; + return S_OK; +} + +STDMETHODIMP CDecoder::ReleaseInStream() +{ + _inStream.Release(); + return S_OK; +} +#endif + +HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress) +{ + RINOK(SetOutStreamSizeResume(outSize)); + return CodeSpec(_inStream, outStream, progress); +} + +}} diff --git a/CPP/7zip/Compress/Lz4Decoder.h b/CPP/7zip/Compress/Lz4Decoder.h new file mode 100644 index 00000000..0de8a2eb --- /dev/null +++ b/CPP/7zip/Compress/Lz4Decoder.h @@ -0,0 +1,99 @@ +// (C) 2016 Tino Reichardt + +/** + * you can define LZ4_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7) + * /TR 2016-10-01 + */ + +#define LZ4_STATIC_LINKING_ONLY +#include "../../../C/Alloc.h" +#include "../../../C/Threads.h" +#include "../../../C/lz4/lz4.h" +#include "../../../C/zstdmt/lz4mt.h" + +#include "../../Windows/System.h" +#include "../../Common/Common.h" +#include "../../Common/MyCom.h" +#include "../ICoder.h" +#include "../Common/StreamUtils.h" +#include "../Common/RegisterCodec.h" +#include "../Common/ProgressMt.h" + +struct Lz4Stream { + ISequentialInStream *inStream; + ISequentialOutStream *outStream; + ICompressProgressInfo *progress; + UInt64 *processedIn; + UInt64 *processedOut; + CCriticalSection *cs; + int flags; +}; + +extern int Lz4Read(void *Stream, LZ4MT_Buffer * in); +extern int Lz4Write(void *Stream, LZ4MT_Buffer * in); + +namespace NCompress { +namespace NLZ4 { + +struct DProps +{ + DProps() { clear (); } + void clear () + { + memset(this, 0, sizeof (*this)); + _ver_major = LZ4_VERSION_MAJOR; + _ver_minor = LZ4_VERSION_MINOR; + _level = 1; + } + + Byte _ver_major; + Byte _ver_minor; + Byte _level; + Byte _reserved[2]; +}; + +class CDecoder:public ICompressCoder, + public ICompressSetDecoderProperties2, + public CMyUnknownImp +{ + CMyComPtr < ISequentialInStream > _inStream; + + DProps _props; + CCriticalSection cs; + + UInt64 _processedIn; + UInt64 _processedOut; + UInt32 _inputSize; + UInt32 _numThreads; + + HRESULT CDecoder::ErrorOut(size_t code); + HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); + HRESULT SetOutStreamSizeResume(const UInt64 *outSize); + +public: + + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) +#ifndef NO_READ_FROM_CODER + MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) +#endif + MY_QUERYINTERFACE_END + + MY_ADDREF_RELEASE + STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size); + STDMETHOD (SetOutStreamSize)(const UInt64 *outSize); + STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads); + +#ifndef NO_READ_FROM_CODER + STDMETHOD (SetInStream)(ISequentialInStream *inStream); + STDMETHOD (ReleaseInStream)(); + UInt64 GetInputProcessedSize() const { return _processedIn; } +#endif + HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress); + + CDecoder(); + virtual ~CDecoder(); +}; + +}} diff --git a/CPP/7zip/Compress/Lz4Encoder.cpp b/CPP/7zip/Compress/Lz4Encoder.cpp new file mode 100644 index 00000000..74d41b4b --- /dev/null +++ b/CPP/7zip/Compress/Lz4Encoder.cpp @@ -0,0 +1,139 @@ +// (C) 2016 Tino Reichardt + +#include "StdAfx.h" +#include "Lz4Encoder.h" +#include "Lz4Decoder.h" + +#ifndef EXTRACT_ONLY +namespace NCompress { +namespace NLZ4 { + +CEncoder::CEncoder(): + _processedIn(0), + _processedOut(0), + _inputSize(0), + _ctx(NULL), + _numThreads(NWindows::NSystem::GetNumberOfProcessors()) +{ + _props.clear(); +} + +CEncoder::~CEncoder() +{ + if (_ctx) + LZ4MT_freeCCtx(_ctx); +} + +STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) +{ + _props.clear(); + + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT & prop = coderProps[i]; + PROPID propID = propIDs[i]; + UInt32 v = (UInt32)prop.ulVal; + switch (propID) + { + case NCoderPropID::kLevel: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + + _props._level = static_cast < Byte > (prop.ulVal); + Byte lz4_level = static_cast < Byte > (LZ4MT_LEVEL_MAX); + if (_props._level > lz4_level) + _props._level = lz4_level; + + break; + } + case NCoderPropID::kNumThreads: + { + SetNumberOfThreads(v); + break; + } + default: + { + break; + } + } + } + + return S_OK; +} + +STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream) +{ + return WriteStream(outStream, &_props, sizeof (_props)); +} + +STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, + ISequentialOutStream *outStream, const UInt64 * /*inSize*/ , + const UInt64 * /*outSize */, ICompressProgressInfo *progress) +{ + LZ4MT_RdWr_t rdwr; + size_t result; + HRESULT res = S_OK; + + struct Lz4Stream Rd; + Rd.inStream = inStream; + Rd.outStream = outStream; + Rd.processedIn = &_processedIn; + Rd.processedOut = &_processedOut; + + struct Lz4Stream Wr; + if (_processedIn == 0) + Wr.progress = progress; + else + Wr.progress = 0; + Wr.inStream = inStream; + Wr.outStream = outStream; + Wr.processedIn = &_processedIn; + Wr.processedOut = &_processedOut; + + /* 1) setup read/write functions */ + rdwr.fn_read = ::Lz4Read; + rdwr.fn_write = ::Lz4Write; + rdwr.arg_read = (void *)&Rd; + rdwr.arg_write = (void *)&Wr; + + /* 2) create compression context, if needed */ + if (!_ctx) + _ctx = LZ4MT_createCCtx(_numThreads, _props._level, _inputSize); + if (!_ctx) + return S_FALSE; + + /* 3) compress */ + result = LZ4MT_CompressCCtx(_ctx, &rdwr); + if (result == (size_t)-LZ4MT_error_read_fail) + res = E_ABORT; + else if (LZ4MT_isError(result)) + if (result != LZ4MT_error_read_fail) + return ErrorOut(result); + + return res; +} + +STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) +{ + const UInt32 kNumThreadsMax = LZ4MT_THREAD_MAX; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + _numThreads = numThreads; + return S_OK; +} + +HRESULT CEncoder::ErrorOut(size_t code) +{ + const char *strError = LZ4MT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + +}} +#endif diff --git a/CPP/7zip/Compress/Lz4Encoder.h b/CPP/7zip/Compress/Lz4Encoder.h new file mode 100644 index 00000000..2faeb696 --- /dev/null +++ b/CPP/7zip/Compress/Lz4Encoder.h @@ -0,0 +1,63 @@ +// (C) 2016 Tino Reichardt + +#define LZ4_STATIC_LINKING_ONLY +#include "../../../C/Alloc.h" +#include "../../../C/Threads.h" +#include "../../../C/lz4/lz4.h" +#include "../../../C/zstdmt/lz4mt.h" + +#include "../../Common/Common.h" +#include "../../Common/MyCom.h" +#include "../ICoder.h" +#include "../Common/StreamUtils.h" + +#ifndef EXTRACT_ONLY +namespace NCompress { +namespace NLZ4 { + +struct CProps +{ + CProps() { clear (); } + void clear () + { + memset(this, 0, sizeof (*this)); + _ver_major = LZ4_VERSION_MAJOR; + _ver_minor = LZ4_VERSION_MINOR; + _level = 3; + } + + Byte _ver_major; + Byte _ver_minor; + Byte _level; + Byte _reserved[2]; +}; + +class CEncoder: + public ICompressCoder, + public ICompressSetCoderProperties, + public ICompressWriteCoderProperties, + public CMyUnknownImp +{ + CProps _props; + + UInt64 _processedIn; + UInt64 _processedOut; + UInt32 _inputSize; + UInt32 _numThreads; + + LZ4MT_CCtx *_ctx; + HRESULT CEncoder::ErrorOut(size_t code); + +public: + MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties) + STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads); + + CEncoder(); + virtual ~CEncoder(); +}; + +}} +#endif diff --git a/CPP/7zip/Compress/Lz4Register.cpp b/CPP/7zip/Compress/Lz4Register.cpp new file mode 100644 index 00000000..a6c237a8 --- /dev/null +++ b/CPP/7zip/Compress/Lz4Register.cpp @@ -0,0 +1,17 @@ +// (C) 2016 Tino Reichardt + +#include "StdAfx.h" + +#include "../Common/RegisterCodec.h" + +#include "Lz4Decoder.h" + +#ifndef EXTRACT_ONLY +#include "Lz4Encoder.h" +#endif + +REGISTER_CODEC_E( + LZ4, + NCompress::NLZ4::CDecoder(), + NCompress::NLZ4::CEncoder(), + 0x4F71104, "LZ4") diff --git a/CPP/7zip/Compress/Lz5Decoder.cpp b/CPP/7zip/Compress/Lz5Decoder.cpp new file mode 100644 index 00000000..a5a5017f --- /dev/null +++ b/CPP/7zip/Compress/Lz5Decoder.cpp @@ -0,0 +1,178 @@ +// (C) 2016 Tino Reichardt + +#include "StdAfx.h" +#include "Lz5Decoder.h" + +int Lz5Read(void *arg, LZ5MT_Buffer * in) +{ + struct Lz5Stream *x = (struct Lz5Stream*)arg; + size_t size = in->size; + + HRESULT res = ReadStream(x->inStream, in->buf, &size); + if (res != S_OK) + return -1; + + in->size = size; + *x->processedIn += size; + + return 0; +} + +int Lz5Write(void *arg, LZ5MT_Buffer * out) +{ + struct Lz5Stream *x = (struct Lz5Stream*)arg; + UInt32 todo = (UInt32)out->size; + UInt32 done = 0; + + while (todo != 0) + { + UInt32 block; + HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); + done += block; + if (res == k_My_HRESULT_WritingWasCut) + break; + if (res != S_OK) + return -1; + if (block == 0) + return E_FAIL; + todo -= block; + } + + *x->processedOut += done; + if (x->progress) + x->progress->SetRatioInfo(x->processedIn, x->processedOut); + + return 0; +} + +namespace NCompress { +namespace NLZ5 { + +CDecoder::CDecoder(): + _processedIn(0), + _processedOut(0), + _inputSize(0), + _numThreads(NWindows::NSystem::GetNumberOfProcessors()) +{ + _props.clear(); +} + +CDecoder::~CDecoder() +{ +} + +HRESULT CDecoder::ErrorOut(size_t code) +{ + const char *strError = LZ5MT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + +STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) +{ + DProps *pProps = (DProps *)prop; + + if (size != sizeof(DProps)) + return E_FAIL; + + memcpy(&_props, pProps, sizeof (DProps)); + + return S_OK; +} + +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) +{ + const UInt32 kNumThreadsMax = LZ5MT_THREAD_MAX; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + _numThreads = numThreads; + return S_OK; +} + +HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/) +{ + _processedOut = 0; + return S_OK; +} + +STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize) +{ + _processedIn = 0; + RINOK(SetOutStreamSizeResume(outSize)); + return S_OK; +} + +HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, + ISequentialOutStream * outStream, ICompressProgressInfo * progress) +{ + LZ5MT_RdWr_t rdwr; + size_t result; + HRESULT res = S_OK; + + struct Lz5Stream Rd; + Rd.inStream = inStream; + Rd.processedIn = &_processedIn; + + struct Lz5Stream Wr; + Wr.progress = progress; + Wr.outStream = outStream; + Wr.processedIn = &_processedIn; + Wr.processedOut = &_processedOut; + + /* 1) setup read/write functions */ + rdwr.fn_read = ::Lz5Read; + rdwr.fn_write = ::Lz5Write; + rdwr.arg_read = (void *)&Rd; + rdwr.arg_write = (void *)&Wr; + + /* 2) create compression context */ + LZ5MT_DCtx *ctx = LZ5MT_createDCtx(_numThreads, _inputSize); + if (!ctx) + return S_FALSE; + + /* 3) compress */ + result = LZ5MT_DecompressDCtx(ctx, &rdwr); + if (result == (size_t)-LZ5MT_error_read_fail) + res = E_ABORT; + else if (LZ5MT_isError(result)) + if (result != LZ5MT_error_read_fail) + return ErrorOut(result); + + /* 4) free resources */ + LZ5MT_freeDCtx(ctx); + return res; +} + +STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream, + const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress) +{ + SetOutStreamSize(outSize); + return CodeSpec(inStream, outStream, progress); +} + +#ifndef NO_READ_FROM_CODER +STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream) +{ + _inStream = inStream; + return S_OK; +} + +STDMETHODIMP CDecoder::ReleaseInStream() +{ + _inStream.Release(); + return S_OK; +} +#endif + +HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress) +{ + RINOK(SetOutStreamSizeResume(outSize)); + return CodeSpec(_inStream, outStream, progress); +} + +}} diff --git a/CPP/7zip/Compress/Lz5Decoder.h b/CPP/7zip/Compress/Lz5Decoder.h new file mode 100644 index 00000000..f6c92b8f --- /dev/null +++ b/CPP/7zip/Compress/Lz5Decoder.h @@ -0,0 +1,99 @@ +// (C) 2016 Tino Reichardt + +/** + * you can define LZ5_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7) + * /TR 2016-10-01 + */ + +#define LZ5_STATIC_LINKING_ONLY +#include "../../../C/Alloc.h" +#include "../../../C/Threads.h" +#include "../../../C/lz5/lz5.h" +#include "../../../C/zstdmt/lz5mt.h" + +#include "../../Windows/System.h" +#include "../../Common/Common.h" +#include "../../Common/MyCom.h" +#include "../ICoder.h" +#include "../Common/StreamUtils.h" +#include "../Common/RegisterCodec.h" +#include "../Common/ProgressMt.h" + +struct Lz5Stream { + ISequentialInStream *inStream; + ISequentialOutStream *outStream; + ICompressProgressInfo *progress; + UInt64 *processedIn; + UInt64 *processedOut; + CCriticalSection *cs; + int flags; +}; + +extern int Lz5Read(void *Stream, LZ5MT_Buffer * in); +extern int Lz5Write(void *Stream, LZ5MT_Buffer * in); + +namespace NCompress { +namespace NLZ5 { + +struct DProps +{ + DProps() { clear (); } + void clear () + { + memset(this, 0, sizeof (*this)); + _ver_major = LZ5_VERSION_MAJOR; + _ver_minor = LZ5_VERSION_MINOR; + _level = 1; + } + + Byte _ver_major; + Byte _ver_minor; + Byte _level; + Byte _reserved[2]; +}; + +class CDecoder:public ICompressCoder, + public ICompressSetDecoderProperties2, + public CMyUnknownImp +{ + CMyComPtr < ISequentialInStream > _inStream; + + DProps _props; + CCriticalSection cs; + + UInt64 _processedIn; + UInt64 _processedOut; + UInt32 _inputSize; + UInt32 _numThreads; + + HRESULT CDecoder::ErrorOut(size_t code); + HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); + HRESULT SetOutStreamSizeResume(const UInt64 *outSize); + +public: + + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) +#ifndef NO_READ_FROM_CODER + MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) +#endif + MY_QUERYINTERFACE_END + + MY_ADDREF_RELEASE + STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size); + STDMETHOD (SetOutStreamSize)(const UInt64 *outSize); + STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads); + +#ifndef NO_READ_FROM_CODER + STDMETHOD (SetInStream)(ISequentialInStream *inStream); + STDMETHOD (ReleaseInStream)(); + UInt64 GetInputProcessedSize() const { return _processedIn; } +#endif + HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress); + + CDecoder(); + virtual ~CDecoder(); +}; + +}} diff --git a/CPP/7zip/Compress/Lz5Encoder.cpp b/CPP/7zip/Compress/Lz5Encoder.cpp new file mode 100644 index 00000000..38e7fe52 --- /dev/null +++ b/CPP/7zip/Compress/Lz5Encoder.cpp @@ -0,0 +1,138 @@ +// (C) 2016 Tino Reichardt + +#include "StdAfx.h" +#include "Lz5Encoder.h" +#include "Lz5Decoder.h" + +#ifndef EXTRACT_ONLY +namespace NCompress { +namespace NLZ5 { + +CEncoder::CEncoder(): + _processedIn(0), + _processedOut(0), + _inputSize(0), + _ctx(NULL), + _numThreads(NWindows::NSystem::GetNumberOfProcessors()) +{ + _props.clear(); +} + +CEncoder::~CEncoder() +{ + if (_ctx) + LZ5MT_freeCCtx(_ctx); +} + +STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) +{ + _props.clear(); + + for (UInt32 i = 0; i < numProps; i++) + { + const PROPVARIANT & prop = coderProps[i]; + PROPID propID = propIDs[i]; + UInt32 v = (UInt32)prop.ulVal; + switch (propID) + { + case NCoderPropID::kLevel: + { + if (prop.vt != VT_UI4) + return E_INVALIDARG; + + _props._level = static_cast < Byte > (prop.ulVal); + Byte lz5_level = static_cast < Byte > (LZ5MT_LEVEL_MAX); + if (_props._level > lz5_level) + _props._level = lz5_level; + + break; + } + case NCoderPropID::kNumThreads: + { + SetNumberOfThreads(v); + break; + } + default: + { + break; + } + } + } + + return S_OK; +} + +STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream) +{ + return WriteStream(outStream, &_props, sizeof (_props)); +} + +STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, + ISequentialOutStream *outStream, const UInt64 * /*inSize*/ , + const UInt64 * /*outSize */, ICompressProgressInfo *progress) +{ + LZ5MT_RdWr_t rdwr; + size_t result; + HRESULT res = S_OK; + + struct Lz5Stream Rd; + Rd.inStream = inStream; + Rd.outStream = outStream; + Rd.processedIn = &_processedIn; + Rd.processedOut = &_processedOut; + + struct Lz5Stream Wr; + if (_processedIn == 0) + Wr.progress = progress; + else + Wr.progress = 0; + Wr.inStream = inStream; + Wr.outStream = outStream; + Wr.processedIn = &_processedIn; + Wr.processedOut = &_processedOut; + + /* 1) setup read/write functions */ + rdwr.fn_read = ::Lz5Read; + rdwr.fn_write = ::Lz5Write; + rdwr.arg_read = (void *)&Rd; + rdwr.arg_write = (void *)&Wr; + + /* 2) create compression context, if needed */ + if (!_ctx) + _ctx = LZ5MT_createCCtx(_numThreads, _props._level, _inputSize); + if (!_ctx) + return S_FALSE; + + /* 3) compress */ + result = LZ5MT_CompressCCtx(_ctx, &rdwr); + if (result == (size_t)-LZ5MT_error_read_fail) + res = E_ABORT; + if (LZ5MT_isError(result)) + return ErrorOut(result); + + return res; +} + +STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) +{ + const UInt32 kNumThreadsMax = LZ5MT_THREAD_MAX; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + _numThreads = numThreads; + return S_OK; +} + +HRESULT CEncoder::ErrorOut(size_t code) +{ + const char *strError = LZ5MT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; +} + +}} +#endif diff --git a/CPP/7zip/Compress/Lz5Encoder.h b/CPP/7zip/Compress/Lz5Encoder.h new file mode 100644 index 00000000..f35a1806 --- /dev/null +++ b/CPP/7zip/Compress/Lz5Encoder.h @@ -0,0 +1,63 @@ +// (C) 2016 Tino Reichardt + +#define LZ5_STATIC_LINKING_ONLY +#include "../../../C/Alloc.h" +#include "../../../C/Threads.h" +#include "../../../C/lz5/lz5.h" +#include "../../../C/zstdmt/lz5mt.h" + +#include "../../Common/Common.h" +#include "../../Common/MyCom.h" +#include "../ICoder.h" +#include "../Common/StreamUtils.h" + +#ifndef EXTRACT_ONLY +namespace NCompress { +namespace NLZ5 { + +struct CProps +{ + CProps() { clear (); } + void clear () + { + memset(this, 0, sizeof (*this)); + _ver_major = LZ5_VERSION_MAJOR; + _ver_minor = LZ5_VERSION_MINOR; + _level = 3; + } + + Byte _ver_major; + Byte _ver_minor; + Byte _level; + Byte _reserved[2]; +}; + +class CEncoder: + public ICompressCoder, + public ICompressSetCoderProperties, + public ICompressWriteCoderProperties, + public CMyUnknownImp +{ + CProps _props; + + UInt64 _processedIn; + UInt64 _processedOut; + UInt32 _inputSize; + UInt32 _numThreads; + + LZ5MT_CCtx *_ctx; + HRESULT CEncoder::ErrorOut(size_t code); + +public: + MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties) + STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads); + + CEncoder(); + virtual ~CEncoder(); +}; + +}} +#endif diff --git a/CPP/7zip/Compress/Lz5Register.cpp b/CPP/7zip/Compress/Lz5Register.cpp new file mode 100644 index 00000000..461fe23d --- /dev/null +++ b/CPP/7zip/Compress/Lz5Register.cpp @@ -0,0 +1,17 @@ +// (C) 2016 Tino Reichardt + +#include "StdAfx.h" + +#include "../Common/RegisterCodec.h" + +#include "Lz5Decoder.h" + +#ifndef EXTRACT_ONLY +#include "Lz5Encoder.h" +#endif + +REGISTER_CODEC_E( + LZ5, + NCompress::NLZ5::CDecoder(), + NCompress::NLZ5::CEncoder(), + 0x4F71105, "LZ5") diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp index 43d11cad..d0e7d658 100644 --- a/CPP/7zip/Compress/ZstdDecoder.cpp +++ b/CPP/7zip/Compress/ZstdDecoder.cpp @@ -1,33 +1,76 @@ -// ZstdDecoder.cpp // (C) 2016 Tino Reichardt #include "StdAfx.h" #include "ZstdDecoder.h" +int ZstdRead(void *arg, ZSTDMT_Buffer * in) +{ + struct ZstdStream *x = (struct ZstdStream*)arg; + size_t size = in->size; + + HRESULT res = ReadStream(x->inStream, in->buf, &size); + if (res != S_OK) + return -1; + + in->size = size; + *x->processedIn += size; + + return 0; +} + +int ZstdWrite(void *arg, ZSTDMT_Buffer * out) +{ + struct ZstdStream *x = (struct ZstdStream*)arg; + UInt32 todo = (UInt32)out->size; + UInt32 done = 0; + + while (todo != 0) + { + UInt32 block; + HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); + done += block; + if (res == k_My_HRESULT_WritingWasCut) + break; + if (res != S_OK) + return -1; + if (block == 0) + return E_FAIL; + todo -= block; + } + + *x->processedOut += done; + if (x->progress) + x->progress->SetRatioInfo(x->processedIn, x->processedOut); + + return 0; +} + namespace NCompress { namespace NZSTD { CDecoder::CDecoder(): - _dstream(NULL), - _buffIn(NULL), - _buffOut(NULL), - _buffInSizeAllocated(0), - _buffOutSizeAllocated(0), - _buffInSize(ZSTD_DStreamInSize()), - _buffOutSize(ZSTD_DStreamOutSize()*4), _processedIn(0), - _processedOut(0) + _processedOut(0), + _inputSize(0), + _numThreads(NWindows::NSystem::GetNumberOfProcessors()) { _props.clear(); } CDecoder::~CDecoder() { - if (_dstream) - ZSTD_freeDStream(_dstream); +} - MyFree(_buffIn); - MyFree(_buffOut); +HRESULT CDecoder::ErrorOut(size_t code) +{ + const char *strError = ZSTDMT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ + + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); + + return S_FALSE; } STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) @@ -37,97 +80,23 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) if (size != sizeof(DProps)) return E_FAIL; -#ifdef ZSTD_LEGACY_SUPPORT - /* version 0.x and 1.x are okay */ - if (pProps->_ver_major > 1) - return E_FAIL; - - /* 0.5, 0.6, 0.7, 0.8 are supported! */ - if (pProps->_ver_major == 0) { - switch (pProps->_ver_minor) { - case 5: - break; - case 6: - break; - case 7: - break; - case 8: - break; - default: - return E_FAIL; - }} -#else - /* only exact version is okay */ - if (pProps->_ver_major != ZSTD_VERSION_MAJOR) - return E_FAIL; - if (pProps->_ver_minor != ZSTD_VERSION_MINOR) - return E_FAIL; -#endif - memcpy(&_props, pProps, sizeof (DProps)); return S_OK; } -HRESULT CDecoder::ErrorOut(size_t code) +STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) { - const char *strError = ZSTD_getErrorName(code); - size_t strErrorLen = strlen(strError) + 1; - wchar_t *wstrError = (wchar_t *)MyAlloc(sizeof(wchar_t) * strErrorLen); - - if (!wstrError) - return E_FAIL; - - mbstowcs(wstrError, strError, strErrorLen - 1); - MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); - MyFree(wstrError); - - return E_FAIL; -} - -HRESULT CDecoder::CreateDecompressor() -{ - size_t result; - - if (!_dstream) { - _dstream = ZSTD_createDStream(); - if (!_dstream) - return E_FAIL; - } - - result = ZSTD_initDStream(_dstream); - if (ZSTD_isError(result)) - return ErrorOut(result); - - /* allocate buffers */ - if (_buffInSizeAllocated != _buffInSize) - { - MyFree(_buffIn); - _buffIn = MyAlloc(_buffInSize); - - if (!_buffIn) - return E_OUTOFMEMORY; - _buffInSizeAllocated = _buffInSize; - } - - if (_buffOutSizeAllocated != _buffOutSize) - { - MyFree(_buffOut); - _buffOut = MyAlloc(_buffOutSize); - - if (!_buffOut) - return E_OUTOFMEMORY; - _buffOutSizeAllocated = _buffOutSize; - } - + const UInt32 kNumThreadsMax = ZSTDMT_THREAD_MAX; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + _numThreads = numThreads; return S_OK; } HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/) { _processedOut = 0; - RINOK(CreateDecompressor()); - return S_OK; } @@ -135,73 +104,48 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize) { _processedIn = 0; RINOK(SetOutStreamSizeResume(outSize)); - return S_OK; } -STDMETHODIMP CDecoder::SetInBufSize(UInt32, UInt32 size) +HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, + ISequentialOutStream * outStream, ICompressProgressInfo * progress) { - _buffInSize = size; - return S_OK; -} - -STDMETHODIMP CDecoder::SetOutBufSize(UInt32, UInt32 size) -{ - _buffOutSize = size; - return S_OK; -} - -HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress) -{ - RINOK(CreateDecompressor()); - + ZSTDMT_RdWr_t rdwr; size_t result; - UInt32 const toRead = static_cast < const UInt32 > (_buffInSize); - for(;;) { - UInt32 read; + HRESULT res = S_OK; - /* read input */ - RINOK(inStream->Read(_buffIn, toRead, &read)); - size_t InSize = static_cast < size_t > (read); - _processedIn += InSize; + struct ZstdStream Rd; + Rd.inStream = inStream; + Rd.processedIn = &_processedIn; - if (InSize == 0) - return S_OK; + struct ZstdStream Wr; + Wr.progress = progress; + Wr.outStream = outStream; + Wr.processedIn = &_processedIn; + Wr.processedOut = &_processedOut; - /* decompress input */ - ZSTD_inBuffer input = { _buffIn, InSize, 0 }; - for (;;) { - ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 }; - result = ZSTD_decompressStream(_dstream, &output , &input); - #if 0 - printf("%s in=%d out=%d result=%d in.pos=%d in.size=%d\n", __FUNCTION__, - InSize, output.pos, result, input.pos, input.size); - fflush(stdout); - #endif - if (ZSTD_isError(result)) - return ErrorOut(result); + /* 1) setup read/write functions */ + rdwr.fn_read = ::ZstdRead; + rdwr.fn_write = ::ZstdWrite; + rdwr.arg_read = (void *)&Rd; + rdwr.arg_write = (void *)&Wr; - /* write decompressed stream and update progress */ - RINOK(WriteStream(outStream, _buffOut, output.pos)); - _processedOut += output.pos; - RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); + /* 2) create decompression context */ + ZSTDMT_DCtx *ctx = ZSTDMT_createDCtx(_numThreads, _inputSize); + if (!ctx) + return S_FALSE; - /* one more round */ - if ((input.pos == input.size) && (result == 1) && output.pos) - continue; + /* 3) decompress */ + result = ZSTDMT_decompressDCtx(ctx, &rdwr); + //printf("decompress = %d / %d\n", result, ZSTDMT_error_read_fail); + if (result == (size_t)-ZSTDMT_error_read_fail) + res = E_ABORT; + else if (ZSTDMT_isError(result)) + return ErrorOut(result); - /* finished */ - if (input.pos == input.size) - break; - - /* end of frame */ - if (result == 0) { - result = ZSTD_initDStream(_dstream); - if (ZSTD_isError(result)) - return ErrorOut(result); - } - } - } + /* 4) free resources */ + ZSTDMT_freeDCtx(ctx); + return res; } STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream, @@ -221,69 +165,14 @@ STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream) STDMETHODIMP CDecoder::ReleaseInStream() { _inStream.Release(); - return S_OK; } - -STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize) -{ - if (processedSize) - *processedSize = 0; - - size_t result; - - if (!_dstream) - if (CreateDecompressor() != S_OK) - return E_FAIL; - - UInt32 read, toRead = static_cast < UInt32 > (_buffInSize); - Byte *dataout = static_cast < Byte* > (data); - for(;;) { - /* read input */ - RINOK(_inStream->Read(_buffIn, toRead, &read)); - size_t InSize = static_cast < size_t > (read); - _processedIn += InSize; - - if (InSize == 0) { - return S_OK; - } - - /* decompress input */ - ZSTD_inBuffer input = { _buffIn, InSize, 0 }; - for (;;) { - ZSTD_outBuffer output = { dataout, _buffOutSize, 0 }; - result = ZSTD_decompressStream(_dstream, &output , &input); - if (ZSTD_isError(result)) - return ErrorOut(result); - - if (processedSize) - *processedSize += static_cast < UInt32 > (output.pos); - - dataout += output.pos; - - /* one more round */ - if ((input.pos == input.size) && (result == 1) && output.pos) - continue; - - /* finished */ - if (input.pos == input.size) - break; - - /* end of frame */ - if (result == 0) { - result = ZSTD_initDStream(_dstream); - if (ZSTD_isError(result)) - return ErrorOut(result); - } - } - } -} +#endif HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress) { RINOK(SetOutStreamSizeResume(outSize)); return CodeSpec(_inStream, outStream, progress); } -#endif }} diff --git a/CPP/7zip/Compress/ZstdDecoder.h b/CPP/7zip/Compress/ZstdDecoder.h index ca63cb42..394554b7 100644 --- a/CPP/7zip/Compress/ZstdDecoder.h +++ b/CPP/7zip/Compress/ZstdDecoder.h @@ -1,22 +1,36 @@ -// ZstdDecoder.h // (C) 2016 Tino Reichardt /** - * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible - * with these versions: 0.5, 0.6, 0.7, 0.8 (0.8 == 1.0) - * - * /TR 2016-09-04 + * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7) + * /TR 2016-10-01 */ #define ZSTD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" -#include "../../../C/ZStd/zstd.h" +#include "../../../C/Threads.h" +#include "../../../C/zstd/zstd.h" +#include "../../../C/zstdmt/zstdmt.h" +#include "../../Windows/System.h" #include "../../Common/Common.h" #include "../../Common/MyCom.h" #include "../ICoder.h" #include "../Common/StreamUtils.h" #include "../Common/RegisterCodec.h" +#include "../Common/ProgressMt.h" + +struct ZstdStream { + ISequentialInStream *inStream; + ISequentialOutStream *outStream; + ICompressProgressInfo *progress; + UInt64 *processedIn; + UInt64 *processedOut; + CCriticalSection *cs; + int flags; +}; + +extern int ZstdRead(void *Stream, ZSTDMT_Buffer * in); +extern int ZstdWrite(void *Stream, ZSTDMT_Buffer * in); namespace NCompress { namespace NZSTD { @@ -39,60 +53,44 @@ struct DProps }; class CDecoder:public ICompressCoder, - public ICompressSetDecoderProperties2, public ICompressSetBufSize, -#ifndef NO_READ_FROM_CODER - public ICompressSetInStream, - public ICompressSetOutStreamSize, public ISequentialInStream, -#endif + public ICompressSetDecoderProperties2, public CMyUnknownImp { CMyComPtr < ISequentialInStream > _inStream; DProps _props; - - ZSTD_DStream *_dstream; - void *_buffIn; - void *_buffOut; - - size_t _buffInSize; - size_t _buffOutSize; - size_t _buffInSizeAllocated; - size_t _buffOutSizeAllocated; + CCriticalSection cs; UInt64 _processedIn; UInt64 _processedOut; + UInt32 _inputSize; + UInt32 _numThreads; - HRESULT CDecoder::CreateDecompressor(); HRESULT CDecoder::ErrorOut(size_t code); HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); HRESULT SetOutStreamSizeResume(const UInt64 *outSize); public: - MY_QUERYINTERFACE_BEGIN2 (ICompressCoder) - MY_QUERYINTERFACE_ENTRY (ICompressSetDecoderProperties2) - MY_QUERYINTERFACE_ENTRY (ICompressSetBufSize) + MY_QUERYINTERFACE_BEGIN2(ICompressCoder) + MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2) #ifndef NO_READ_FROM_CODER - MY_QUERYINTERFACE_ENTRY (ICompressSetInStream) - MY_QUERYINTERFACE_ENTRY (ICompressSetOutStreamSize) - MY_QUERYINTERFACE_ENTRY (ISequentialInStream) + MY_QUERYINTERFACE_ENTRY(ICompressSetInStream) #endif MY_QUERYINTERFACE_END MY_ADDREF_RELEASE STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); - STDMETHOD (SetDecoderProperties2) (const Byte *data, UInt32 size); - STDMETHOD (SetOutStreamSize) (const UInt64 *outSize); - STDMETHOD (SetInBufSize) (UInt32 streamIndex, UInt32 size); - STDMETHOD (SetOutBufSize) (UInt32 streamIndex, UInt32 size); + STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size); + STDMETHOD (SetOutStreamSize)(const UInt64 *outSize); + STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads); #ifndef NO_READ_FROM_CODER - STDMETHOD (SetInStream) (ISequentialInStream *inStream); - STDMETHOD (ReleaseInStream) (); - STDMETHOD (Read) (void *data, UInt32 size, UInt32 *processedSize); - HRESULT CodeResume (ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress); - UInt64 GetInputProcessedSize () const { return _processedIn; } + STDMETHOD (SetInStream)(ISequentialInStream *inStream); + STDMETHOD (ReleaseInStream)(); + UInt64 GetInputProcessedSize() const { return _processedIn; } #endif + HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress); CDecoder(); virtual ~CDecoder(); diff --git a/CPP/7zip/Compress/ZstdEncoder.cpp b/CPP/7zip/Compress/ZstdEncoder.cpp index b882a1a4..428726fc 100644 --- a/CPP/7zip/Compress/ZstdEncoder.cpp +++ b/CPP/7zip/Compress/ZstdEncoder.cpp @@ -1,32 +1,27 @@ -// ZstdEncoder.cpp // (C) 2016 Tino Reichardt #include "StdAfx.h" #include "ZstdEncoder.h" +#include "ZstdDecoder.h" #ifndef EXTRACT_ONLY namespace NCompress { namespace NZSTD { CEncoder::CEncoder(): - _cstream(NULL), - _buffIn(NULL), - _buffOut(NULL), - _buffInSize(0), - _buffOutSize(0), _processedIn(0), - _processedOut(0) + _processedOut(0), + _inputSize(0), + _ctx(NULL), + _numThreads(NWindows::NSystem::GetNumberOfProcessors()) { _props.clear(); } CEncoder::~CEncoder() { - if (_cstream) - ZSTD_freeCStream(_cstream); - - MyFree(_buffIn); - MyFree(_buffOut); + if (_ctx) + ZSTDMT_freeCCtx(_ctx); } STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) @@ -37,6 +32,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI { const PROPVARIANT & prop = coderProps[i]; PROPID propID = propIDs[i]; + UInt32 v = (UInt32)prop.ulVal; switch (propID) { case NCoderPropID::kLevel: @@ -52,6 +48,11 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI break; } + case NCoderPropID::kNumThreads: + { + SetNumberOfThreads(v); + break; + } default: { break; @@ -59,9 +60,6 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI } } - _processedIn = 0; - _processedOut = 0; - return S_OK; } @@ -71,75 +69,70 @@ STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream) } STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, - ISequentialOutStream *outStream, const UInt64 * /* inSize */ , - const UInt64 * /* outSize */ , ICompressProgressInfo *progress) + ISequentialOutStream *outStream, const UInt64 * /*inSize*/ , + const UInt64 * /*outSize */, ICompressProgressInfo *progress) { + ZSTDMT_RdWr_t rdwr; size_t result; + HRESULT res = S_OK; - /* init only once in beginning */ - if (!_cstream) { + struct ZstdStream Rd; + Rd.inStream = inStream; + Rd.outStream = outStream; + Rd.processedIn = &_processedIn; + Rd.processedOut = &_processedOut; - /* allocate stream */ - _cstream = ZSTD_createCStream(); - if (!_cstream) - return E_OUTOFMEMORY; + struct ZstdStream Wr; + if (_processedIn == 0) + Wr.progress = progress; + else + Wr.progress = 0; + Wr.inStream = inStream; + Wr.outStream = outStream; + Wr.processedIn = &_processedIn; + Wr.processedOut = &_processedOut; - /* allocate buffers */ - _buffInSize = ZSTD_CStreamInSize(); - _buffIn = MyAlloc(_buffInSize); - if (!_buffIn) - return E_OUTOFMEMORY; + /* 1) setup read/write functions */ + rdwr.fn_read = ::ZstdRead; + rdwr.fn_write = ::ZstdWrite; + rdwr.arg_read = (void *)&Rd; + rdwr.arg_write = (void *)&Wr; - _buffOutSize = ZSTD_CStreamOutSize(); - _buffOut = MyAlloc(_buffOutSize); - if (!_buffOut) - return E_OUTOFMEMORY; - } - - /* init or re-init stream */ - result = ZSTD_initCStream(_cstream, _props._level); - if (ZSTD_isError(result)) + /* 2) create compression context, if needed */ + if (!_ctx) + _ctx = ZSTDMT_createCCtx(_numThreads, _props._level, _inputSize); + if (!_ctx) return S_FALSE; - UInt32 read, toRead = static_cast < UInt32 > (_buffInSize); - for(;;) { + /* 3) compress */ + result = ZSTDMT_compressCCtx(_ctx, &rdwr); + if (result == (size_t)-ZSTDMT_error_read_fail) + res = E_ABORT; + else if (ZSTDMT_isError(result)) + return ErrorOut(result); - /* read input */ - RINOK(inStream->Read(_buffIn, toRead, &read)); - size_t InSize = static_cast < size_t > (read); - _processedIn += InSize; + return res; +} - if (InSize == 0) { +STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) +{ + const UInt32 kNumThreadsMax = ZSTDMT_THREAD_MAX; + if (numThreads < 1) numThreads = 1; + if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; + _numThreads = numThreads; + return S_OK; +} - /* @eof */ - ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 }; - result = ZSTD_endStream(_cstream, &output); - if (ZSTD_isError(result)) - return S_FALSE; +HRESULT CEncoder::ErrorOut(size_t code) +{ + const char *strError = ZSTDMT_getErrorString(code); + wchar_t wstrError[200+5]; /* no malloc here, /TR */ - if (output.pos) { - /* write last compressed bytes and update progress */ - RINOK(WriteStream(outStream, _buffOut, output.pos)); - _processedOut += output.pos; - RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); - } + mbstowcs(wstrError, strError, 200); + MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK); + MyFree(wstrError); - return S_OK; - } - - /* compress input */ - ZSTD_inBuffer input = { _buffIn, InSize, 0 }; - while (input.pos < input.size) { - ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 }; - result = ZSTD_compressStream(_cstream, &output , &input); - if (ZSTD_isError(result)) - return S_FALSE; - /* write compressed stream and update progress */ - RINOK(WriteStream(outStream, _buffOut, output.pos)); - _processedOut += output.pos; - RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); - } - } + return S_FALSE; } }} diff --git a/CPP/7zip/Compress/ZstdEncoder.h b/CPP/7zip/Compress/ZstdEncoder.h index b91f6d6a..41cf815d 100644 --- a/CPP/7zip/Compress/ZstdEncoder.h +++ b/CPP/7zip/Compress/ZstdEncoder.h @@ -1,9 +1,10 @@ -// ZstdEncoder.h // (C) 2016 Tino Reichardt #define ZSTD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" -#include "../../../C/ZStd/zstd.h" +#include "../../../C/Threads.h" +#include "../../../C/zstd/zstd.h" +#include "../../../C/zstdmt/zstdmt.h" #include "../../Common/Common.h" #include "../../Common/MyCom.h" @@ -39,21 +40,20 @@ class CEncoder: { CProps _props; - ZSTD_CStream *_cstream; - void *_buffIn; - void *_buffOut; - size_t _buffInSize; - size_t _buffOutSize; UInt64 _processedIn; UInt64 _processedOut; + UInt32 _inputSize; + UInt32 _numThreads; - HRESULT CreateCompressor(); + ZSTDMT_CCtx *_ctx; + HRESULT CEncoder::ErrorOut(size_t code); public: MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties) - STDMETHOD (Code) (ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); - STDMETHOD (SetCoderProperties) (const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); - STDMETHOD (WriteCoderProperties) (ISequentialOutStream *outStream); + STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress); + STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps); + STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream); + STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads); CEncoder(); virtual ~CEncoder(); diff --git a/CPP/7zip/Compress/ZstdRegister.cpp b/CPP/7zip/Compress/ZstdRegister.cpp index 1fcdbb67..94abdd74 100644 --- a/CPP/7zip/Compress/ZstdRegister.cpp +++ b/CPP/7zip/Compress/ZstdRegister.cpp @@ -1,4 +1,3 @@ -// ZstdRegister.cpp // (C) 2016 Tino Reichardt #include "StdAfx.h" diff --git a/CPP/7zip/Guid.txt b/CPP/7zip/Guid.txt index c7463167..3ad13f43 100644 --- a/CPP/7zip/Guid.txt +++ b/CPP/7zip/Guid.txt @@ -161,8 +161,11 @@ Handler GUIDs: 0A lzma 0B lzma86 0C xz - 0D ppmd + 0E zstd + 0F lz4 + 10 lz5 + C6 Lzip C7 Ext C8 VMDK C9 VDI diff --git a/CPP/7zip/UI/Common/OpenArchive.cpp b/CPP/7zip/UI/Common/OpenArchive.cpp index 968f0caa..79461210 100644 --- a/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/CPP/7zip/UI/Common/OpenArchive.cpp @@ -1066,6 +1066,7 @@ static const char * const k_Formats_with_simple_signuature[] = , "rar" , "bzip2" , "gzip" + , "lzip" , "cab" , "wim" , "rpm" diff --git a/CPP/7zip/UI/Explorer/ContextMenu.cpp b/CPP/7zip/UI/Explorer/ContextMenu.cpp index f789c2ad..16d6b7ab 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -283,8 +283,12 @@ static const char * const kArcExts[] = "7z" , "bz2" , "gz" + , "lz" + , "lz4" + , "lz5" , "rar" , "zip" + , "zst" }; static bool IsItArcExt(const UString &ext) diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index 094d26cf..6d0a564d 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -46,7 +46,7 @@ static const UInt32 kLangIDs[] = IDT_COMPRESS_SOLID, IDT_COMPRESS_THREADS, IDT_COMPRESS_PARAMETERS, - + IDG_COMPRESS_OPTIONS, IDX_COMPRESS_SFX, IDX_COMPRESS_SHARED, @@ -87,13 +87,9 @@ static const UInt32 g_Levels[] = { IDS_METHOD_STORE, IDS_METHOD_FASTEST, - 0, IDS_METHOD_FAST, - 0, IDS_METHOD_NORMAL, - 0, IDS_METHOD_MAXIMUM, - 0, IDS_METHOD_ULTRA }; @@ -101,6 +97,8 @@ enum EMethodID { kCopy, kZSTD, + kLZ4, + kLZ5, kLZMA, kLZMA2, kPPMd, @@ -114,6 +112,8 @@ static const LPCWSTR kMethodsNames[] = { L"Copy", L"ZSTD", + L"LZ4", + L"LZ5", L"LZMA", L"LZMA2", L"PPMd", @@ -126,6 +126,8 @@ static const LPCWSTR kMethodsNames[] = static const EMethodID g_7zMethods[] = { kZSTD, + kLZ4, + kLZ5, kLZMA2, kLZMA, kPPMd, @@ -167,6 +169,21 @@ static const EMethodID g_XzMethods[] = kLZMA2 }; +static const EMethodID g_ZstdMethods[] = +{ + kZSTD +}; + +static const EMethodID g_Lz4Methods[] = +{ + kLZ4 +}; + +static const EMethodID g_Lz5Methods[] = +{ + kLZ5 +}; + static const EMethodID g_SwfcMethods[] = { kDeflate @@ -227,6 +244,24 @@ static const CFormatInfo g_Formats[] = METHODS_PAIR(g_XzMethods), false, false, true, false, false, false }, + { + L"zstd", + (1 << 0) | (1 << 1) | (1 << 5) | (1 << 11) | (1 << 17) | (1 << 22), + METHODS_PAIR(g_ZstdMethods), + false, false, true, false, false, false + }, + { + L"lz4", + (1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 16), + METHODS_PAIR(g_Lz4Methods), + false, false, true, false, false, false + }, + { + L"lz5", + (1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 16), + METHODS_PAIR(g_Lz5Methods), + false, false, true, false, false, false + }, { L"Swfc", (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9), @@ -281,7 +316,7 @@ static const NCompressDialog::NUpdateMode::kFresh, NCompressDialog::NUpdateMode::kSync }; - + static const UInt32 k_UpdateMode_IDs[] = { IDS_COMPRESS_UPDATE_MODE_ADD, @@ -324,7 +359,7 @@ void CCompressDialog::GetButton_Bools(UINT id, CBoolPair &b1, CBoolPair &b2) b1.Val = b2.Val = val; } - + bool CCompressDialog::OnInit() { #ifdef LANG @@ -346,7 +381,7 @@ bool CCompressDialog::OnInit() m_Order.Attach(GetItem(IDC_COMPRESS_ORDER)); m_Solid.Attach(GetItem(IDC_COMPRESS_SOLID)); m_NumThreads.Attach(GetItem(IDC_COMPRESS_THREADS)); - + m_UpdateMode.Attach(GetItem(IDC_COMPRESS_UPDATE_MODE)); m_PathMode.Attach(GetItem(IDC_COMPRESS_PATH_MODE)); @@ -358,7 +393,7 @@ bool CCompressDialog::OnInit() m_RegistryInfo.Load(); CheckButton(IDX_PASSWORD_SHOW, m_RegistryInfo.ShowPassword); CheckButton(IDX_COMPRESS_ENCRYPT_FILE_NAMES, m_RegistryInfo.EncryptHeaders); - + CheckButton_TwoBools(IDX_COMPRESS_NT_SYM_LINKS, Info.SymLinks, m_RegistryInfo.SymLinks); CheckButton_TwoBools(IDX_COMPRESS_NT_HARD_LINKS, Info.HardLinks, m_RegistryInfo.HardLinks); CheckButton_TwoBools(IDX_COMPRESS_NT_ALT_STREAMS, Info.AltStreams, m_RegistryInfo.AltStreams); @@ -396,7 +431,7 @@ bool CCompressDialog::OnInit() } SetLevel(); SetParams(); - + for (unsigned i = 0; i < m_RegistryInfo.ArcPaths.Size() && i < kHistorySize; i++) m_ArchivePath.AddString(m_RegistryInfo.ArcPaths[i]); @@ -517,7 +552,7 @@ void CCompressDialog::CheckControlsEnable() bool multiThreadEnable = fi.MultiThread; Info.MultiThreadIsAllowed = multiThreadEnable; Info.EncryptHeadersIsAllowed = fi.EncryptFileNames; - + EnableItem(IDC_COMPRESS_SOLID, fi.Solid); EnableItem(IDC_COMPRESS_THREADS, multiThreadEnable); @@ -525,7 +560,7 @@ void CCompressDialog::CheckControlsEnable() { const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - + ShowItem_Bool(IDX_COMPRESS_NT_SYM_LINKS, ai.Flags_SymLinks()); ShowItem_Bool(IDX_COMPRESS_NT_HARD_LINKS, ai.Flags_HardLinks()); ShowItem_Bool(IDX_COMPRESS_NT_ALT_STREAMS, ai.Flags_AltStreams()); @@ -725,12 +760,12 @@ void CCompressDialog::OnOK() ShowErrorMessage(*this, k_IncorrectPathMessage); return; } - + m_RegistryInfo.ArcPaths.Clear(); AddUniqueString(m_RegistryInfo.ArcPaths, s); Info.ArcPath = s; } - + Info.UpdateMode = (NCompressDialog::NUpdateMode::EEnum)k_UpdateMode_Vals[m_UpdateMode.GetCurSel()];; Info.PathMode = (NWildcard::ECensorPathMode)k_PathMode_Vals[m_PathMode.GetCurSel()]; @@ -770,12 +805,12 @@ void CCompressDialog::OnOK() } m_Params.GetText(Info.Options); - + UString volumeString; m_Volume.GetText(volumeString); volumeString.Trim(); Info.VolumeSizes.Clear(); - + if (!volumeString.IsEmpty()) { if (!ParseVolumeSizes(volumeString, Info.VolumeSizes)) @@ -804,16 +839,16 @@ void CCompressDialog::OnOK() sTemp.Trim(); AddUniqueString(m_RegistryInfo.ArcPaths, sTemp); } - + if (m_RegistryInfo.ArcPaths.Size() > kHistorySize) m_RegistryInfo.ArcPaths.DeleteBack(); - + if (Info.FormatIndex >= 0) m_RegistryInfo.ArcType = (*ArcFormats)[Info.FormatIndex].Name; m_RegistryInfo.ShowPassword = IsShowPasswordChecked(); m_RegistryInfo.Save(); - + CModalDialog::OnOK(); } @@ -867,10 +902,6 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) } case IDC_COMPRESS_LEVEL: { - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - int index = FindRegistryFormatAlways(ai.Name); - NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - fo.ResetForLevelChange(); SetMethod(GetMethodID()); SetSolidBlockSize(); SetNumThreads(); @@ -880,13 +911,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) } case IDC_COMPRESS_METHOD: { - // MessageBoxW(*this, L"IDC_COMPRESS_METHOD!", L"7-Zip", MB_ICONERROR); - if (GetMethodID() == kZSTD) - { - SetLevel_zstd(); - } else { - SetLevel_default(); - } + SetLevel(); SetDictionary(); SetOrder(); SetSolidBlockSize(); @@ -1018,10 +1043,13 @@ void CCompressDialog::SetNearestSelectComboBox(NControl::CComboBox &comboBox, UI comboBox.SetCurSel(0); } -void CCompressDialog::SetLevel_zstd() +void CCompressDialog::SetLevel() { UInt32 level = GetLevel2(); + UInt32 LevelsMask; + UInt32 langID = 0; + SetMethod(GetMethodID()); const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; { int index = FindRegistryFormat(ai.Name); @@ -1030,92 +1058,45 @@ void CCompressDialog::SetLevel_zstd() const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; if (fo.Level <= 22) level = fo.Level; + else + level = 5; } } - /* ZStandard has 22 levels */ m_Level.ResetContent(); + if (GetMethodID() == kZSTD) + LevelsMask = g_Formats[6].LevelsMask; + else if (GetMethodID() == kLZ4) + LevelsMask = g_Formats[7].LevelsMask; + else if (GetMethodID() == kLZ5) + LevelsMask = g_Formats[8].LevelsMask; + else + LevelsMask = g_Formats[GetStaticFormatIndex()].LevelsMask; + for (unsigned i = 0; i <= 22; i++) { TCHAR s[40]; TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 }; - ConvertUInt32ToString(i, s); - lstrcat(t, s); - switch (i) { - case 0: - lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(IDS_METHOD_STORE)); - lstrcat(t, TEXT(")")); - break; - case 1: - lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(IDS_METHOD_FASTEST)); - lstrcat(t, TEXT(")")); - break; - case 5: - lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(IDS_METHOD_FAST)); - lstrcat(t, TEXT(")")); - break; - case 11: - lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(IDS_METHOD_NORMAL)); - lstrcat(t, TEXT(")")); - break; - case 17: - lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(IDS_METHOD_MAXIMUM)); - lstrcat(t, TEXT(")")); - break; - case 22: - lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(IDS_METHOD_ULTRA)); - lstrcat(t, TEXT(")")); - break; - } - int index = (int)m_Level.AddString(t); - m_Level.SetItemData(index, i); - } - SetNearestSelectComboBox(m_Level, level); - return; -} + // max reached + if (LevelsMask < (UInt32)(1 << i)) + break; -void CCompressDialog::SetLevel_default() -{ - UInt32 level = GetLevel2(); - - const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - { - int index = FindRegistryFormat(ai.Name); - if (index >= 0) + if ((LevelsMask & (1 << i)) != 0) { - const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; - if (fo.Level <= 9) - level = fo.Level; - else - level = 9; - } - } - - /* 9 default levels */ - m_Level.ResetContent(); - const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; - - for (unsigned i = 0; i <= 9; i++) - { - if ((fi.LevelsMask & (1 << i)) != 0) - { - TCHAR s[40]; - TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 }; - UInt32 langID = g_Levels[i]; ConvertUInt32ToString(i, s); lstrcat(t, s); lstrcat(t, TEXT(" (")); - lstrcat(t, LangString(langID)); + lstrcat(t, LangString(g_Levels[langID])); lstrcat(t, TEXT(")")); int index = (int)m_Level.AddString(t); m_Level.SetItemData(index, i); + langID++; + } else { + ConvertUInt32ToString(i, s); + lstrcat(t, s); + int index = (int)m_Level.AddString(t); + m_Level.SetItemData(index, i); } } @@ -1123,18 +1104,6 @@ void CCompressDialog::SetLevel_default() return; } -void CCompressDialog::SetLevel() -{ - SetMethod(); - - if (GetMethodID() == kZSTD) - { - SetLevel_zstd(); - } else { - SetLevel_default(); - } -} - void CCompressDialog::SetMethod(int keepMethodId) { m_Method.ResetContent(); @@ -1156,7 +1125,7 @@ void CCompressDialog::SetMethod(int keepMethodId) } bool isSfx = IsSFX(); bool weUseSameMethod = false; - + for (unsigned m = 0; m < fi.NumMethods; m++) { EMethodID methodID = fi.MethodIDs[m]; @@ -1266,21 +1235,21 @@ void CCompressDialog::SetDictionary() const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; int index = FindRegistryFormat(ai.Name); UInt32 defaultDict = (UInt32)(Int32)-1; - + if (index >= 0) { const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; if (fo.Method.IsEqualTo_NoCase(GetMethodSpec())) defaultDict = fo.Dictionary; } - + int methodID = GetMethodID(); UInt32 level = GetLevel2(); if (methodID < 0) return; UInt64 maxRamSize; bool maxRamSize_Defined = GetMaxRamSizeForProgram(maxRamSize); - + switch (methodID) { case kLZMA: @@ -1295,17 +1264,17 @@ void CCompressDialog::SetDictionary() else if (level >= 3) defaultDict = (1 << 20); else defaultDict = (kMinDicSize); } - + AddDictionarySize(kMinDicSize); m_Dictionary.SetCurSel(0); - + for (unsigned i = 20; i <= 31; i++) for (unsigned j = 0; j < 2; j++) { if (i == 20 && j > 0) continue; UInt32 dict = ((UInt32)(2 + j) << (i - 1)); - + if (dict > #ifdef MY_CPU_64BIT (3 << 29) @@ -1314,7 +1283,7 @@ void CCompressDialog::SetDictionary() #endif ) continue; - + AddDictionarySize(dict); UInt64 decomprSize; UInt64 requiredComprSize = GetMemoryUsage(dict, decomprSize); @@ -1325,7 +1294,7 @@ void CCompressDialog::SetDictionary() // SetNearestSelectComboBox(m_Dictionary, defaultDict); break; } - + case kPPMd: { if (defaultDict == (UInt32)(Int32)-1) @@ -1357,7 +1326,7 @@ void CCompressDialog::SetDictionary() || m_Dictionary.GetCount() == 1) m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); } - + // SetNearestSelectComboBox(m_Dictionary, defaultDict); break; } @@ -1368,14 +1337,14 @@ void CCompressDialog::SetDictionary() m_Dictionary.SetCurSel(0); break; } - + case kDeflate64: { AddDictionarySize(64 << 10); m_Dictionary.SetCurSel(0); break; } - + case kBZip2: { if (defaultDict == (UInt32)(Int32)-1) @@ -1384,7 +1353,7 @@ void CCompressDialog::SetDictionary() else if (level >= 3) defaultDict = (500 << 10); else defaultDict = (100 << 10); } - + for (unsigned i = 1; i <= 9; i++) { UInt32 dict = ((UInt32)i * 100) << 10; @@ -1392,15 +1361,15 @@ void CCompressDialog::SetDictionary() if (dict <= defaultDict || m_Dictionary.GetCount() == 0) m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); } - + break; } - + case kPPMdZip: { if (defaultDict == (UInt32)(Int32)-1) defaultDict = (1 << (19 + (level > 8 ? 8 : level))); - + for (unsigned i = 20; i <= 28; i++) { UInt32 dict = (1 << i); @@ -1411,7 +1380,7 @@ void CCompressDialog::SetDictionary() || m_Dictionary.GetCount() == 1) m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1); } - + // SetNearestSelectComboBox(m_Dictionary, defaultDict); break; } @@ -1449,7 +1418,7 @@ void CCompressDialog::SetOrder() const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; int index = FindRegistryFormat(ai.Name); UInt32 defaultOrder = (UInt32)(Int32)-1; - + if (index >= 0) { const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; @@ -1461,7 +1430,7 @@ void CCompressDialog::SetOrder() UInt32 level = GetLevel2(); if (methodID < 0) return; - + switch (methodID) { case kLZMA: @@ -1480,7 +1449,7 @@ void CCompressDialog::SetOrder() SetNearestSelectComboBox(m_Order, defaultOrder); break; } - + case kPPMd: { if (defaultOrder == (UInt32)(Int32)-1) @@ -1490,10 +1459,10 @@ void CCompressDialog::SetOrder() else if (level >= 5) defaultOrder = 6; else defaultOrder = 4; } - + AddOrder(2); AddOrder(3); - + for (unsigned i = 2; i < 8; i++) for (unsigned j = 0; j < 4; j++) { @@ -1501,12 +1470,12 @@ void CCompressDialog::SetOrder() if (order < 32) AddOrder(order); } - + AddOrder(32); SetNearestSelectComboBox(m_Order, defaultOrder); break; } - + case kDeflate: case kDeflate64: { @@ -1516,7 +1485,7 @@ void CCompressDialog::SetOrder() else if (level >= 7) defaultOrder = 64; else defaultOrder = 32; } - + for (unsigned i = 3; i <= 8; i++) for (unsigned j = 0; j < 2; j++) { @@ -1524,15 +1493,15 @@ void CCompressDialog::SetOrder() if (order <= 256) AddOrder(order); } - + AddOrder(methodID == kDeflate64 ? 257 : 258); SetNearestSelectComboBox(m_Order, defaultOrder); break; } - + case kBZip2: break; - + case kPPMdZip: { if (defaultOrder == (UInt32)(Int32)-1) @@ -1592,9 +1561,9 @@ void CCompressDialog::SetSolidBlockSize() m_Solid.SetItemData(index, (UInt32)kNoSolidBlockSize); m_Solid.SetCurSel(0); } - + bool needSet = (defaultBlockSize == (UInt32)(Int32)-1); - + for (unsigned i = 20; i <= 36; i++) { if (needSet && dict >= (((UInt64)1 << (i - 7))) && i <= 32) @@ -1607,12 +1576,12 @@ void CCompressDialog::SetSolidBlockSize() int index = (int)m_Solid.AddString(s); m_Solid.SetItemData(index, (UInt32)i); } - + { int index = (int)m_Solid.AddString(LangString(IDS_COMPRESS_SOLID)); m_Solid.SetItemData(index, kSolidBlockSize); } - + if (defaultBlockSize == (UInt32)(Int32)-1) defaultBlockSize = kSolidBlockSize; if (defaultBlockSize != kNoSolidBlockSize) @@ -1645,6 +1614,9 @@ void CCompressDialog::SetNumThreads() int methodID = GetMethodID(); switch (methodID) { + case kLZ4: numAlgoThreadsMax = 128; break; + case kLZ5: numAlgoThreadsMax = 128; break; + case kZSTD: numAlgoThreadsMax = 128; break; case kLZMA: numAlgoThreadsMax = 2; break; case kLZMA2: numAlgoThreadsMax = 32; break; case kBZip2: numAlgoThreadsMax = 32; break; @@ -1676,7 +1648,7 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) if (fi.Filter && level >= 9) size += (12 << 20) * 2 + (5 << 20); UInt32 numThreads = GetNumThreads2(); - + if (IsZipFormat()) { UInt32 numSubThreads = 1; @@ -1686,9 +1658,9 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) if (numMainThreads > 1) size += (UInt64)numMainThreads << 25; } - + int methidId = GetMethodID(); - + switch (methidId) { case kLZMA: @@ -1716,9 +1688,9 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) size1 += (2 << 20) + (4 << 20); numThreads1 = 2; } - + UInt32 numBlockThreads = numThreads / numThreads1; - + if (methidId == kLZMA || numBlockThreads == 1) size1 += (UInt64)dict * 3 / 2; else @@ -1734,45 +1706,13 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) decompressMemory = dict + (2 << 20); return size; } - + case kPPMd: { decompressMemory = dict + (2 << 20); return size + decompressMemory; } - case kZSTD: - { - /* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=131075 */ - size = 0; - switch (level) { - case 1: size = 824228; decompressMemory = 415024; return size; - case 2: size = 1282980; decompressMemory = 415024; return size; - case 3: size = 922532; decompressMemory = 415024; return size; - case 4: size = 1414052; decompressMemory = 415024; return size; - case 5: size = 1545124; decompressMemory = 415024; return size; - case 6: size = 1807268; decompressMemory = 415024; return size; - case 7: size = 1807268; decompressMemory = 415024; return size; - case 8: size = 1807268; decompressMemory = 415024; return size; - case 9: size = 1807268; decompressMemory = 415024; return size; - case 10: size = 1807268; decompressMemory = 415024; return size; - case 11: size = 2331556; decompressMemory = 415024; return size; - case 12: size = 2331556; decompressMemory = 415024; return size; - case 13: size = 3380132; decompressMemory = 415024; return size; - case 14: size = 3004832; decompressMemory = 415024; return size; - case 15: size = 3004832; decompressMemory = 415024; return size; - case 16: size = 4697834; decompressMemory = 415024; return size; - case 17: size = 4697834; decompressMemory = 415024; return size; - case 18: size = 4697834; decompressMemory = 415024; return size; - case 19: size = 4697834; decompressMemory = 415024; return size; - case 20: size = 4697834; decompressMemory = 415024; return size; - case 21: size = 4697834; decompressMemory = 415024; return size; - case 22: size = 4697834; decompressMemory = 415024; return size; - } - decompressMemory = 0; - return size; - } - case kDeflate: case kDeflate64: { @@ -1785,21 +1725,21 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory) decompressMemory = (2 << 20); return size; } - + case kBZip2: { decompressMemory = (7 << 20); UInt64 memForOneThread = (10 << 20); return size + memForOneThread * numThreads; } - + case kPPMdZip: { decompressMemory = dict + (2 << 20); return size + (UInt64)decompressMemory * numThreads; } } - + return (UInt64)(Int64)-1; } @@ -1821,7 +1761,7 @@ void CCompressDialog::PrintMemUsage(UINT res, UInt64 value) lstrcat(s, TEXT(" MB")); SetItemText(res, s); } - + void CCompressDialog::SetMemoryUsage() { UInt64 decompressMem; diff --git a/CPP/7zip/UI/GUI/CompressDialog.h b/CPP/7zip/UI/GUI/CompressDialog.h index 1944aa83..1c0e6c61 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.h +++ b/CPP/7zip/UI/GUI/CompressDialog.h @@ -130,8 +130,6 @@ class CCompressDialog: public NWindows::NControl::CModalDialog void SetNearestSelectComboBox(NWindows::NControl::CComboBox &comboBox, UInt32 value); void SetLevel(); - void SetLevel_zstd(); - void SetLevel_default(); void SetMethod(int keepMethodId = -1); int GetMethodID(); diff --git a/CPP/7zip/zstd.mak b/CPP/7zip/zstd.mak deleted file mode 100644 index 26124bd7..00000000 --- a/CPP/7zip/zstd.mak +++ /dev/null @@ -1,13 +0,0 @@ - -COMPRESS_OBJS = $(COMPRESS_OBJS) \ - $O\ZstdDecoder.obj \ - $O\ZstdEncoder.obj \ - $O\ZstdRegister.obj \ - -ZSTD_OBJS = \ - $O\entropy_common.obj \ - $O\fse_decompress.obj \ - $O\huf_decompress.obj \ - $O\zstd_common.obj \ - $O\zstd_decompress.obj \ - $O\xxhash.obj \ diff --git a/CPP/build-x32.cmd b/CPP/build-x32.cmd index e1b0badb..e9d7c423 100644 --- a/CPP/build-x32.cmd +++ b/CPP/build-x32.cmd @@ -31,9 +31,13 @@ cd %ROOT%\Bundles\SFXWin nmake %OPTS% copy O\7z.sfx %OUTDIR%\7z.sfx -cd %ROOT%\Bundles\Codec_zstd +cd %ROOT%\Bundles\Codec_lz4 nmake %OPTS% -copy O\zstd.dll %OUTDIR%\zstd-x32min.dll +copy O\lz4.dll %OUTDIR%\lz4-x32.dll + +cd %ROOT%\Bundles\Codec_lz5 +nmake %OPTS% +copy O\lz5.dll %OUTDIR%\lz5-x32.dll cd %ROOT%\Bundles\Codec_zstdF nmake %OPTS% diff --git a/CPP/build-x64.cmd b/CPP/build-x64.cmd index 92c838f3..44d22573 100644 --- a/CPP/build-x64.cmd +++ b/CPP/build-x64.cmd @@ -31,9 +31,13 @@ cd %ROOT%\Bundles\SFXWin nmake %OPTS% copy AMD64\7z.sfx %OUTDIR%\7z.sfx -cd %ROOT%\Bundles\Codec_zstd +cd %ROOT%\Bundles\Codec_lz4 nmake %OPTS% -copy AMD64\zstd.dll %OUTDIR%\zstd-x64min.dll +copy AMD64\lz4.dll %OUTDIR%\lz4-x64.dll + +cd %ROOT%\Bundles\Codec_lz5 +nmake %OPTS% +copy AMD64\lz5.dll %OUTDIR%\lz5-x64.dll cd %ROOT%\Bundles\Codec_zstdf nmake %OPTS% diff --git a/CPP/build-x64x.cmd b/CPP/build-x64x.cmd new file mode 100644 index 00000000..360d22fd --- /dev/null +++ b/CPP/build-x64x.cmd @@ -0,0 +1,65 @@ +@echo on + +set ROOT=%cd%\7zip +set OUTDIR=%ROOT%\bin64 +mkdir %OUTDIR% + +set OPTS=CPU=AMD64 MY_STATIC_LINK=1 +set LFLAGS=/SUBSYSTEM:WINDOWS,"5.02" + +cd %ROOT%\Bundles\Format7z +nmake %OPTS% +copy AMD64\7za.dll %OUTDIR%\7za.dll + +cd %ROOT%\Bundles\Format7zF +nmake %OPTS% +copy AMD64\7z.dll %OUTDIR%\7z.dll +goto ende + +cd %ROOT%\UI\FileManager +nmake %OPTS% +copy AMD64\7zFM.exe %OUTDIR%\7zFM.exe + +cd %ROOT%\UI\GUI +nmake %OPTS% +copy AMD64\7zG.exe %OUTDIR%\7zG.exe + +cd %ROOT%\UI\Explorer +nmake %OPTS% +copy AMD64\7-zip.dll %OUTDIR%\7-zip.dll + +cd %ROOT%\Bundles\SFXWin +nmake %OPTS% +copy AMD64\7z.sfx %OUTDIR%\7z.sfx + +cd %ROOT%\Bundles\Codec_lz4 +nmake %OPTS% +copy AMD64\lz4.dll %OUTDIR%\lz4-x64.dll + +cd %ROOT%\Bundles\Codec_lz5 +nmake %OPTS% +copy AMD64\lz5.dll %OUTDIR%\lz5-x64.dll + +cd %ROOT%\Bundles\Codec_zstdf +nmake %OPTS% +copy AMD64\zstd.dll %OUTDIR%\zstd-x64.dll + +cd %ROOT%\..\..\C\Util\7zipInstall +nmake %OPTS% +copy AMD64\7zipInstall.exe %OUTDIR%\Install-x64.exe + +cd %ROOT%\..\..\C\Util\7zipUninstall +nmake %OPTS% +copy AMD64\7zipUninstall.exe %OUTDIR%\Uninstall.exe + +set LFLAGS=/SUBSYSTEM:CONSOLE,"5.02" +cd %ROOT%\UI\Console +nmake %OPTS% +copy AMD64\7z.exe %OUTDIR%\7z.exe + +cd %ROOT%\Bundles\SFXCon +nmake %OPTS% +copy AMD64\7zCon.sfx %OUTDIR%\7zCon.sfx + +:ende +cd %ROOT%\.. diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi index 542609ad..a8c17ed9 100644 --- a/DOC/7zip.nsi +++ b/DOC/7zip.nsi @@ -3,7 +3,7 @@ !define VERSION_MAJOR 16 !define VERSION_MINOR 04 -!define VERSION_POSTFIX_FULL "" +!define VERSION_POSTFIX_FULL "-ZS" !ifdef WIN64 !ifdef IA64 !define VERSION_SYS_POSTFIX_FULL " for Windows IA-64" @@ -534,6 +534,7 @@ Section Uninstall DeleteRegKey HKCR 7-Zip.hfs DeleteRegKey HKCR 7-Zip.iso DeleteRegKey HKCR 7-Zip.lha + DeleteRegKey HKCR 7-Zip.lz DeleteRegKey HKCR 7-Zip.lzh DeleteRegKey HKCR 7-Zip.lzma DeleteRegKey HKCR 7-Zip.ntfs @@ -545,6 +546,7 @@ Section Uninstall DeleteRegKey HKCR 7-Zip.tar DeleteRegKey HKCR 7-Zip.taz DeleteRegKey HKCR 7-Zip.tgz + DeleteRegKey HKCR 7-Zip.tlz DeleteRegKey HKCR 7-Zip.tpz DeleteRegKey HKCR 7-Zip.txz DeleteRegKey HKCR 7-Zip.vhd diff --git a/DOC/Methods.txt b/DOC/Methods.txt index 3ca90577..ca78a348 100644 --- a/DOC/Methods.txt +++ b/DOC/Methods.txt @@ -123,7 +123,10 @@ List of defined IDs 0x xx - reserved 10 xx - reserved (LZHAM) - 11 xx - reserved (Zstd) + 11 xx - reserved (Tino Reichardt) + 11 01 - reserved (Zstd) + 11 04 - reserved (LZ4) + 11 05 - reserved (LZ5) 06.. - Crypto diff --git a/DOC/readme.txt b/DOC/readme.txt index b4ff2f25..995c4bd2 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -123,7 +123,7 @@ Windows common files for Windows related code Bundle Modules that are bundles of other modules (files) - Alone 7za.exe: Standalone version of 7-Zip console that supports only 7z/xz/cab/zip/gzip/bzip2/tar. + Alone 7za.exe: Standalone version of 7-Zip console that supports only 7z/xz/cab/zip/gzip/bzip2/lzip/tar Alone7z 7zr.exe: Standalone version of 7-Zip console that supports only 7z (reduced version) Fm Standalone version of 7-Zip File Manager Format7z 7za.dll: .7z support diff --git a/cleanup.cmd b/cleanup.cmd new file mode 100644 index 00000000..a6b00305 --- /dev/null +++ b/cleanup.cmd @@ -0,0 +1,33 @@ +@echo off + +rd /S /Q "CPP\7zip\bin32" +rd /S /Q "CPP\7zip\bin64" + +rd /S /Q "C/Util/7zipInstall/O" +rd /S /Q "C/Util/7zipUninstall/O" +rd /S /Q "CPP/7zip/Bundles/Codec_lz4/O" +rd /S /Q "CPP/7zip/Bundles/Codec_lz5/O" +rd /S /Q "CPP/7zip/Bundles/Codec_zstdF/O" +rd /S /Q "CPP/7zip/Bundles/Format7z/O" +rd /S /Q "CPP/7zip/Bundles/Format7zF/O" +rd /S /Q "CPP/7zip/Bundles/SFXCon/O" +rd /S /Q "CPP/7zip/Bundles/SFXWin/O" +rd /S /Q "CPP/7zip/UI/Console/O" +rd /S /Q "CPP/7zip/UI/Explorer/O" +rd /S /Q "CPP/7zip/UI/FileManager/O" +rd /S /Q "CPP/7zip/UI/GUI/O" + +rd /S /Q "C/Util/7zipInstall/AMD64" +rd /S /Q "C/Util/7zipUninstall/AMD64" +rd /S /Q "CPP/7zip/Bundles/Codec_lz4/AMD64" +rd /S /Q "CPP/7zip/Bundles/Codec_lz5/AMD64" +rd /S /Q "CPP/7zip/Bundles/Codec_zstdF/AMD64" +rd /S /Q "CPP/7zip/Bundles/Format7z/AMD64" +rd /S /Q "CPP/7zip/Bundles/Format7zF/AMD64" +rd /S /Q "CPP/7zip/Bundles/SFXCon/AMD64" +rd /S /Q "CPP/7zip/Bundles/SFXWin/AMD64" +rd /S /Q "CPP/7zip/UI/Console/AMD64" +rd /S /Q "CPP/7zip/UI/Explorer/AMD64" +rd /S /Q "CPP/7zip/UI/FileManager/AMD64" +rd /S /Q "CPP/7zip/UI/GUI/AMD64" +