diff --git a/C/zstd/error_public.h b/C/zstd/error_public.h index e8cfcc91..29050b3b 100644 --- a/C/zstd/error_public.h +++ b/C/zstd/error_public.h @@ -63,7 +63,11 @@ typedef enum { ZSTD_error_maxCode } ZSTD_ErrorCode; -/* note : compare with size_t function results using ZSTD_getError() */ +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +const char* ZSTD_getErrorString(ZSTD_ErrorCode code); #if defined (__cplusplus) diff --git a/C/zstd/zbuff.h b/C/zstd/zbuff.h index 65f01e56..7820db26 100644 --- a/C/zstd/zbuff.h +++ b/C/zstd/zbuff.h @@ -185,7 +185,7 @@ ZSTDLIB_API ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem); /*--- Advanced Streaming function ---*/ ZSTDLIB_API size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize); + ZSTD_parameters params, unsigned long long pledgedSrcSize); #endif /* ZBUFF_STATIC_LINKING_ONLY */ diff --git a/C/zstd/zbuff_compress.c b/C/zstd/zbuff_compress.c index ef8448ce..837d22cf 100644 --- a/C/zstd/zbuff_compress.c +++ b/C/zstd/zbuff_compress.c @@ -137,7 +137,7 @@ size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize) + ZSTD_parameters params, unsigned long long pledgedSrcSize) { /* allocate buffers */ { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; diff --git a/C/zstd/zstd.h b/C/zstd/zstd.h index de2ec1eb..304edd36 100644 --- a/C/zstd/zstd.h +++ b/C/zstd/zstd.h @@ -61,7 +61,7 @@ extern "C" { ***************************************/ #define ZSTD_VERSION_MAJOR 0 #define ZSTD_VERSION_MINOR 7 -#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_RELEASE 3 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str @@ -197,14 +197,13 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, * Use them only in association with static linking. * ==================================================================================== */ -/*--- Dependency ---*/ -#include "mem.h" /* U32 */ - /*--- Constants ---*/ #define ZSTD_MAGICNUMBER 0xFD2FB527 /* v0.7 */ #define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U -#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? 25 : 27)) +#define ZSTD_WINDOWLOG_MAX_32 25 +#define ZSTD_WINDOWLOG_MAX_64 27 +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) #define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) #define ZSTD_CHAINLOG_MIN 4 @@ -229,19 +228,19 @@ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable f typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt } ZSTD_strategy; /*< from faster to stronger */ typedef struct { - U32 windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ - U32 chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ - U32 hashLog; /*< dispatch table : larger == faster, more memory */ - U32 searchLog; /*< nb of searches : larger == more compression, slower */ - U32 searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ - U32 targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ + unsigned windowLog; /*< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /*< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /*< dispatch table : larger == faster, more memory */ + unsigned searchLog; /*< nb of searches : larger == more compression, slower */ + unsigned searchLength; /*< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /*< acceptable match size for optimal parser (only) : larger == more compression, slower */ ZSTD_strategy strategy; } ZSTD_compressionParameters; typedef struct { - U32 contentSizeFlag; /*< 1: content size will be in frame header (if known). */ - U32 checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ - U32 noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ + unsigned contentSizeFlag; /*< 1: content size will be in frame header (if known). */ + unsigned checksumFlag; /*< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + unsigned noDictIDFlag; /*< 1: no dict ID will be saved into frame header (if dictionary compression) */ } ZSTD_frameParameters; typedef struct { @@ -272,12 +271,12 @@ ZSTDLIB_API unsigned ZSTD_maxCLevel (void); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize); +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level and srcSize. * `srcSize` value is optional, select 0 if not known */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize); /*! ZSTD_checkCParams() : * Ensure param values remain within authorized range */ @@ -286,7 +285,7 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); /*! ZSTD_adjustCParams() : * optimize params for a given `srcSize` and `dictSize`. * both values are optional, select `0` if unknown. */ -ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize); +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : * Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ @@ -299,6 +298,15 @@ ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, /*--- Advanced Decompression functions ---*/ +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*! ZSTD_createDCtx_advanced() : * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); @@ -309,7 +317,7 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ******************************************************************/ ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); -ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx); ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -345,10 +353,10 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci */ typedef struct { - U64 frameContentSize; - U32 windowSize; - U32 dictID; - U32 checksumFlag; + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; } ZSTD_frameParams; ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input */ @@ -408,12 +416,14 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds * Block functions ****************************************/ /*! Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which is non-negligible on very small blocks. User will have to take in charge required information to regenerate data, such as compressed and content sizes. A few rules to respect : - Uncompressed block size must be <= ZSTD_BLOCKSIZE_MAX (128 KB) - + If you need to compress more, it's recommended to use ZSTD_compress() instead, since frame metadata costs become negligible. - - Compressing or decompressing requires a context structure + + If you need to compress more, cut data into multiple blocks + + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + - Compressing and decompressing require a context structure + Use ZSTD_createCCtx() and ZSTD_createDCtx() - It is necessary to init context before starting + compression : ZSTD_compressBegin() @@ -423,23 +433,16 @@ ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t ds - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. In which case, nothing is produced into `dst`. + User must test for such outcome and deal directly with uncompressed data - + ZSTD_decompressBlock() doesn't accept uncompressed data as input !! + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. + Use ZSTD_insertBlock() in such a case. + Insert block once it's copied into its final position. */ #define ZSTD_BLOCKSIZE_MAX (128 * 1024) /* define, for static allocation */ ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); - - -/*-************************************* -* Error management -***************************************/ -#include "error_public.h" -/*! ZSTD_getErrorCode() : - convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); -ZSTDLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful to track uncompressed blocks */ #endif /* ZSTD_STATIC_LINKING_ONLY */ diff --git a/C/zstd/zstd_compress.c b/C/zstd/zstd_compress.c index 03746617..cd55c7ae 100644 --- a/C/zstd/zstd_compress.c +++ b/C/zstd/zstd_compress.c @@ -221,7 +221,7 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz Both `srcSize` and `dictSize` are optional (use 0 if unknown), but if both are 0, no optimization can be done. Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ -ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, U64 srcSize, size_t dictSize) +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) { if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ @@ -2407,7 +2407,7 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, * @return : 0, or an error code */ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_parameters params, U64 pledgedSrcSize) + ZSTD_parameters params, unsigned long long pledgedSrcSize) { /* compression parameters verification and optimization */ { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); @@ -2744,7 +2744,7 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /*! ZSTD_getCParams() : * @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. * Size values are optional, provide 0 if not known or unused */ -ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, size_t dictSize) +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_compressionParameters cp; size_t const addedSize = srcSize ? 0 : 500; @@ -2765,7 +2765,7 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, U64 srcSize, si /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * All fields of `ZSTD_frameParameters` are set to default (0) */ -ZSTD_parameters ZSTD_getParams(int compressionLevel, U64 srcSize, size_t dictSize) { +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { ZSTD_parameters params; ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); memset(¶ms, 0, sizeof(params)); diff --git a/C/zstd/zstd_decompress.c b/C/zstd/zstd_decompress.c index 001a19ae..2645342c 100644 --- a/C/zstd/zstd_decompress.c +++ b/C/zstd/zstd_decompress.c @@ -195,7 +195,7 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) /* Frame format description Frame Header - [ Block Header - Block ] - Frame End 1) Frame Header - - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd_static.h) + - 4 bytes - Magic Number : ZSTD_MAGICNUMBER (defined within zstd.h) - 1 byte - Frame Descriptor 2) Block Header - 3 bytes, starting with a 2-bits descriptor @@ -391,6 +391,26 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t } +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not provided within frame header + - frame header unknown / not supported + - frame header not completely provided (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize); +#endif + { ZSTD_frameParams fparams; + size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + /** ZSTD_decodeFrameHeader() : * `srcSize` must be the size provided by ZSTD_frameHeaderSize(). * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ @@ -629,7 +649,7 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, /* FSE table descriptors */ { U32 const LLtype = *ip >> 6; - U32 const Offtype = (*ip >> 4) & 3; + U32 const OFtype = (*ip >> 4) & 3; U32 const MLtype = (*ip >> 2) & 3; ip++; @@ -637,17 +657,17 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, if (ip > iend-3) return ERROR(srcSize_wrong); /* min : all 3 are "raw", hence no header, but at least xxLog bits per type */ /* Build DTables */ - { size_t const bhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); - if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); - ip += bhSize; + { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; } - { size_t const bhSize = ZSTD_buildSeqTable(DTableOffb, Offtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); - if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); - ip += bhSize; + { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; } - { size_t const bhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); - if (ZSTD_isError(bhSize)) return ERROR(corruption_detected); - ip += bhSize; + { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; } } return ip-istart; @@ -688,42 +708,37 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) 0x2000, 0x4000, 0x8000, 0x10000 }; static const U32 ML_base[MaxML+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 34, 36, 38, 40, 44, 48, 56, 64, 80, 96, 0x80, 0x100, 0x200, 0x400, 0x800, - 0x1000, 0x2000, 0x4000, 0x8000, 0x10000 }; + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; static const U32 OF_base[MaxOff+1] = { - 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, - 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, - 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, - 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, /*fake*/ 1, 1 }; + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; /* sequence */ { size_t offset; if (!ofCode) offset = 0; else { - offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= 26 bits */ + offset = OF_base[ofCode] + BIT_readBits(&(seqState->DStream), ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&(seqState->DStream)); } - if (offset < ZSTD_REP_NUM) { - if (llCode == 0 && offset <= 1) offset = 1-offset; - - if (offset != 0) { - size_t temp = seqState->prevOffset[offset]; - if (offset != 1) { - seqState->prevOffset[2] = seqState->prevOffset[1]; - } + if (ofCode <= 1) { + if ((llCode == 0) & (offset <= 1)) offset = 1-offset; + if (offset) { + size_t const temp = seqState->prevOffset[offset]; + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset = temp; - } else { offset = seqState->prevOffset[0]; } } else { - offset -= ZSTD_REP_MOVE; seqState->prevOffset[2] = seqState->prevOffset[1]; seqState->prevOffset[1] = seqState->prevOffset[0]; seqState->prevOffset[0] = offset; @@ -731,11 +746,11 @@ static seq_t ZSTD_decodeSequence(seqState_t* seqState) seq.offset = offset; } - seq.matchLength = ML_base[mlCode] + MINMATCH + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBits(&(seqState->DStream), mlBits) : 0); /* <= 16 bits */ if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&(seqState->DStream)); seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBits(&(seqState->DStream), llBits) : 0); /* <= 16 bits */ - if (MEM_32bits() | + if (MEM_32bits() || (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&(seqState->DStream)); /* ANS state update */ @@ -924,6 +939,16 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, } +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + size_t ZSTD_generateNxByte(void* dst, size_t dstCapacity, BYTE byte, size_t length) { if (length > dstCapacity) return ERROR(dstSize_tooSmall); @@ -1020,10 +1045,7 @@ size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) - { U32 const magicNumber = MEM_readLE32(src); - if (ZSTD_isLegacy(magicNumber)) - return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize, magicNumber); - } + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); #endif ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); ZSTD_checkContinuity(dctx, dst); @@ -1262,8 +1284,8 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t struct ZSTD_DDict_s { - void* dictContent; - size_t dictContentSize; + void* dict; + size_t dictSize; ZSTD_DCtx* refContext; }; /* typedef'd tp ZSTD_CDict within zstd.h */ @@ -1295,8 +1317,8 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu return NULL; } } - ddict->dictContent = dictContent; - ddict->dictContentSize = dictSize; + ddict->dict = dictContent; + ddict->dictSize = dictSize; ddict->refContext = dctx; return ddict; } @@ -1316,7 +1338,7 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) ZSTD_freeFunction const cFree = ddict->refContext->customMem.customFree; void* const opaque = ddict->refContext->customMem.opaque; ZSTD_freeDCtx(ddict->refContext); - cFree(opaque, ddict->dictContent); + cFree(opaque, ddict->dict); cFree(opaque, ddict); return 0; } @@ -1329,6 +1351,9 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, const void* src, size_t srcSize, const ZSTD_DDict* ddict) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); +#endif return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, dst, dstCapacity, src, srcSize); diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp index ce955a6e..a39b7800 100644 --- a/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/CPP/7zip/Archive/7z/7zHandler.cpp @@ -450,6 +450,32 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const GetStringForSizeValue(dest, GetUi32(props + 1)); } } + #if 1 + else if (id == k_ZSTD) + { + name = "ZSTD"; + if (propsSize == 5) + { + /** + * _ver_major + * _ver_minor + * _level + * _empty[2] + */ + char *dest = s; + *dest++ = 'v'; + ConvertUInt32ToString(props[0], dest); + dest += MyStringLen(dest); + *dest++ = '.'; + ConvertUInt32ToString(props[1], dest); + dest += MyStringLen(dest); + *dest++ = ','; + *dest++ = 'l'; + ConvertUInt32ToString(props[2], dest); + dest += MyStringLen(dest); + } + } + #endif else if (id == k_Delta) { name = "Delta"; diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index 9b2e43ea..5b3bd95d 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -19,7 +19,7 @@ namespace NArchive { namespace N7z { static const char *k_LZMA_Name = "LZMA"; -static const char *kDefaultMethodName = "LZMA2"; +static const char *kDefaultMethodName = "zstd"; static const char *k_Copy_Name = "Copy"; static const char *k_MatchFinder_ForHeaders = "BT2"; diff --git a/CPP/7zip/Archive/7z/7zHeader.h b/CPP/7zip/Archive/7z/7zHeader.h index 813dbe7f..fab413ba 100644 --- a/CPP/7zip/Archive/7z/7zHeader.h +++ b/CPP/7zip/Archive/7z/7zHeader.h @@ -119,7 +119,7 @@ const UInt32 k_ARM = 0x3030501; const UInt32 k_ARMT = 0x3030701; const UInt32 k_SPARC = 0x3030805; -const UInt32 k_ZStd = 0x4F71101; +const UInt32 k_ZSTD = 0x4F71101; const UInt32 k_AES = 0x6F10701; diff --git a/CPP/7zip/UI/GUI/CompressDialog.cpp b/CPP/7zip/UI/GUI/CompressDialog.cpp index bf817515..18be9a5b 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.cpp +++ b/CPP/7zip/UI/GUI/CompressDialog.cpp @@ -100,45 +100,45 @@ static const UInt32 g_Levels[] = enum EMethodID { kCopy, + kZSTD, kLZMA, kLZMA2, kPPMd, kBZip2, kDeflate, kDeflate64, - kPPMdZip, - kZStd + kPPMdZip }; static const LPCWSTR kMethodsNames[] = { L"Copy", + L"ZSTD", L"LZMA", L"LZMA2", L"PPMd", L"BZip2", L"Deflate", L"Deflate64", - L"PPMd", - L"ZStd" + L"PPMd" }; static const EMethodID g_7zMethods[] = { + kZSTD, kLZMA2, kLZMA, kPPMd, - kBZip2, - kZStd + kBZip2 }; static const EMethodID g_7zSfxMethods[] = { kCopy, + kZSTD, kLZMA, kLZMA2, - kPPMd, - kZStd + kPPMd }; static const EMethodID g_ZipMethods[] = @@ -174,7 +174,7 @@ struct CFormatInfo { LPCWSTR Name; UInt32 LevelsMask; - const EMethodID *MathodIDs; + const EMethodID *MethodIDs; unsigned NumMethods; bool Filter; @@ -835,6 +835,8 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) SetItemText(IDT_COMPRESS_ARCHIVE_FOLDER, DirPrefix); /* + + UString path; m_ArchivePath.GetText(path); m_ArchivePath.SetText(L""); @@ -867,7 +869,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) int index = FindRegistryFormatAlways(ai.Name); NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; fo.ResetForLevelChange(); - SetMethod(); + SetMethod(GetMethodID()); SetSolidBlockSize(); SetNumThreads(); CheckSFXNameChange(); @@ -876,6 +878,13 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam) } case IDC_COMPRESS_METHOD: { + // MessageBoxW(*this, L"IDC_COMPRESS_METHOD!", L"7-Zip", MB_ICONERROR); + if (GetMethodID() == kZSTD) + { + SetLevel_zstd(); + } else { + SetLevel_default(); + } SetDictionary(); SetOrder(); SetSolidBlockSize(); @@ -1007,12 +1016,69 @@ void CCompressDialog::SetNearestSelectComboBox(NControl::CComboBox &comboBox, UI comboBox.SetCurSel(0); } -void CCompressDialog::SetLevel() +void CCompressDialog::SetLevel_zstd() { - m_Level.ResetContent(); - const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; + UInt32 level = GetLevel2(); + + const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; + { + int index = FindRegistryFormat(ai.Name); + if (index >= 0) + { + const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index]; + if (fo.Level <= 22) + level = fo.Level; + } + } + + /* ZStandard has 22 levels */ + m_Level.ResetContent(); + for (unsigned i = 0; i <= 22; i++) + { + TCHAR s[40]; + TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 }; + ConvertUInt32ToString(i, s); + lstrcat(t, s); + switch (i) { + case 1: + lstrcat(t, TEXT(" (")); + lstrcat(t, LangString(IDS_METHOD_FASTEST)); + lstrcat(t, TEXT(")")); + break; + case 5: + lstrcat(t, TEXT(" (")); + lstrcat(t, LangString(IDS_METHOD_FAST)); + lstrcat(t, TEXT(")")); + break; + case 11: + lstrcat(t, TEXT(" (")); + lstrcat(t, LangString(IDS_METHOD_NORMAL)); + lstrcat(t, TEXT(")")); + break; + case 17: + lstrcat(t, TEXT(" (")); + lstrcat(t, LangString(IDS_METHOD_MAXIMUM)); + lstrcat(t, TEXT(")")); + break; + case 22: + lstrcat(t, TEXT(" (")); + lstrcat(t, LangString(IDS_METHOD_ULTRA)); + lstrcat(t, TEXT(")")); + break; + } + int index = (int)m_Level.AddString(t); + m_Level.SetItemData(index, i); + } + + SetNearestSelectComboBox(m_Level, level); + return; +} + +void CCompressDialog::SetLevel_default() +{ + UInt32 level = GetLevel2(); + const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()]; - UInt32 level = 5; { int index = FindRegistryFormat(ai.Name); if (index >= 0) @@ -1024,24 +1090,48 @@ void CCompressDialog::SetLevel() level = 9; } } - + + /* 9 default levels */ + m_Level.ResetContent(); + const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()]; + for (unsigned i = 0; i <= 9; i++) { if ((fi.LevelsMask & (1 << i)) != 0) { + TCHAR s[40]; + TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 }; UInt32 langID = g_Levels[i]; - int index = (int)m_Level.AddString(LangString(langID)); + ConvertUInt32ToString(i, s); + lstrcat(t, s); + lstrcat(t, TEXT(" (")); + lstrcat(t, LangString(langID)); + lstrcat(t, TEXT(")")); + int index = (int)m_Level.AddString(t); m_Level.SetItemData(index, i); } } + SetNearestSelectComboBox(m_Level, level); + return; +} + +void CCompressDialog::SetLevel() +{ SetMethod(); + + if (GetMethodID() == kZSTD) + { + SetLevel_zstd(); + } else { + SetLevel_default(); + } } void CCompressDialog::SetMethod(int keepMethodId) { m_Method.ResetContent(); - UInt32 level = GetLevel(); + UInt32 level = GetLevel2(); if (level == 0) { SetDictionary(); @@ -1062,23 +1152,26 @@ void CCompressDialog::SetMethod(int keepMethodId) for (unsigned m = 0; m < fi.NumMethods; m++) { - EMethodID methodID = fi.MathodIDs[m]; + EMethodID methodID = fi.MethodIDs[m]; if (isSfx) if (!IsMethodSupportedBySfx(methodID)) continue; const LPCWSTR method = kMethodsNames[methodID]; int itemIndex = (int)m_Method.AddString(GetSystemString(method)); m_Method.SetItemData(itemIndex, methodID); + + if (keepMethodId == methodID) { m_Method.SetCurSel(itemIndex); weUseSameMethod = true; continue; } + if ((defaultMethod.IsEqualTo_NoCase(method) || m == 0) && !weUseSameMethod) m_Method.SetCurSel(itemIndex); } - + if (!weUseSameMethod) { SetDictionary(); diff --git a/CPP/7zip/UI/GUI/CompressDialog.h b/CPP/7zip/UI/GUI/CompressDialog.h index 1c0e6c61..1944aa83 100644 --- a/CPP/7zip/UI/GUI/CompressDialog.h +++ b/CPP/7zip/UI/GUI/CompressDialog.h @@ -130,6 +130,8 @@ class CCompressDialog: public NWindows::NControl::CModalDialog void SetNearestSelectComboBox(NWindows::NControl::CComboBox &comboBox, UInt32 value); void SetLevel(); + void SetLevel_zstd(); + void SetLevel_default(); void SetMethod(int keepMethodId = -1); int GetMethodID(); diff --git a/README.md b/README.md index 4c19f687..ba313388 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ This is the Github Page of 7-Zip with support for **Zstd**, short for Zstandard, which is a fast lossless compression algorithm, targeting real-time compression scenarios at zlib-level compression ratio. -- ZStd Homepage: https://github.com/Cyan4973/zstd +- Zstd Homepage: https://github.com/Cyan4973/zstd - 7-Zip ZStd Homepage: https://mcmilk.de/projects/7-Zip-ZStd/ - Request for inclusion into the mainline 7-Zip: https://sourceforge.net/p/sevenzip/discussion/45797/thread/a7e4f3f3/