From 00c0d31e31a18b5198a81bbd48b9a800cc1883e1 Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Thu, 15 Sep 2016 17:57:28 +0200 Subject: [PATCH] feature release - you can open / crate .tar.zst files now - direct compressing of files with zstd works also - you can also open files compressed with pzstd - next version will do also threaded compression like pzstd does - zst files can be registred with 7-Zip ZS now - update to latest zstd dev release v1.0.1 - the About Box will give you a hint, that this 7-Zip is not the default one --- C/7zVersion.h | 6 +- C/7zVersionTr.h | 2 +- C/zstd/zbuff_compress.c | 340 ----------------------- C/zstd/zbuff_decompress.c | 297 -------------------- C/zstd/zstd.h | 93 ++++--- C/zstd/zstd_compress.c | 324 ++++++++++++---------- C/zstd/zstd_decompress.c | 339 +++++++++++------------ C/zstd/zstd_internal.h | 26 +- C/zstd/zstd_v05.c | 190 +------------ C/zstd/zstd_v06.c | 205 +------------- C/zstd/zstd_v07.c | 206 +------------- CPP/7zip/Archive/Icons/zst.ico | 0 CPP/7zip/Archive/ZstdHandler.cpp | 367 +++++++++++++++++++++++++ CPP/7zip/Bundles/Format7zF/Arc.mak | 1 + CPP/7zip/Bundles/Format7zF/resource.rc | 4 +- CPP/7zip/Compress/ZstdDecoder.cpp | 21 +- CPP/7zip/Compress/ZstdDecoder.h | 1 - CPP/7zip/UI/Explorer/ContextMenu.cpp | 2 +- 18 files changed, 813 insertions(+), 1611 deletions(-) delete mode 100644 C/zstd/zbuff_compress.c delete mode 100644 C/zstd/zbuff_decompress.c create mode 100644 CPP/7zip/Archive/Icons/zst.ico create mode 100644 CPP/7zip/Archive/ZstdHandler.cpp diff --git a/C/7zVersion.h b/C/7zVersion.h index 6d4fe7c9..bac7c0f0 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,9 +1,9 @@ #define MY_VER_MAJOR 16 #define MY_VER_MINOR 02 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "16.02" -#define MY_VERSION "16.02" -#define MY_DATE "2016-06-12" +#define MY_VERSION_NUMBERS "16.02 ZS" +#define MY_VERSION "16.02 ZS" +#define MY_DATE "2016-09-15" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h index 934a9d6f..776367d8 100644 --- a/C/7zVersionTr.h +++ b/C/7zVersionTr.h @@ -3,7 +3,7 @@ #define MY_VER_BUILD 1 #define MY_VERSION_NUMBERS "1.0.1" #define MY_VERSION "1.0.1" -#define MY_DATE "2016-09-04" +#define MY_DATE "2016-09-15" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Tino Reichardt" diff --git a/C/zstd/zbuff_compress.c b/C/zstd/zbuff_compress.c deleted file mode 100644 index 5d929185..00000000 --- a/C/zstd/zbuff_compress.c +++ /dev/null @@ -1,340 +0,0 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - -/* ************************************* -* Dependencies -***************************************/ -#include -#include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */ -#define ZBUFF_STATIC_LINKING_ONLY -#include "zbuff.h" - - -/* ************************************* -* Constants -***************************************/ -static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE; - - -/*-*********************************************************** -* Streaming compression -* -* A ZBUFF_CCtx object is required to track streaming operation. -* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources. -* Use ZBUFF_compressInit() to start a new compression operation. -* ZBUFF_CCtx objects can be reused multiple times. -* -* Use ZBUFF_compressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input. -* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst . -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency) -* or an error code, which can be tested using ZBUFF_isError(). -* -* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer. -* Note that it will not output more than *dstCapacityPtr. -* Therefore, some content might still be left into its internal buffer if dst buffer is too small. -* @return : nb of bytes still present into internal buffer (0 if it's empty) -* or an error code, which can be tested using ZBUFF_isError(). -* -* ZBUFF_compressEnd() instructs to finish a frame. -* It will perform a flush and write frame epilogue. -* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small. -* @return : nb of bytes still present into internal buffer (0 if it's empty) -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) -* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value. -* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed. -* ***********************************************************/ - -typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage; - -/* *** Resources *** */ -struct ZBUFF_CCtx_s { - ZSTD_CCtx* zc; - char* inBuff; - size_t inBuffSize; - size_t inToCompress; - size_t inBuffPos; - size_t inBuffTarget; - size_t blockSize; - char* outBuff; - size_t outBuffSize; - size_t outBuffContentSize; - size_t outBuffFlushedSize; - ZBUFF_cStage stage; - U32 checksum; - U32 frameEnded; - ZSTD_customMem customMem; -}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */ - -ZBUFF_CCtx* ZBUFF_createCCtx(void) -{ - return ZBUFF_createCCtx_advanced(defaultCustomMem); -} - -ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem) -{ - ZBUFF_CCtx* zbc; - - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx)); - if (zbc==NULL) return NULL; - memset(zbc, 0, sizeof(ZBUFF_CCtx)); - memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem)); - zbc->zc = ZSTD_createCCtx_advanced(customMem); - if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; } - return zbc; -} - -size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc) -{ - if (zbc==NULL) return 0; /* support free on NULL */ - ZSTD_freeCCtx(zbc->zc); - if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); - if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); - zbc->customMem.customFree(zbc->customMem.opaque, zbc); - return 0; -} - - -/* ====== Initialization ====== */ - -size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc, - const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize) -{ - /* allocate buffers */ - { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; - if (zbc->inBuffSize < neededInBuffSize) { - zbc->inBuffSize = neededInBuffSize; - zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */ - zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize); - if (zbc->inBuff == NULL) return ERROR(memory_allocation); - } - zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); - } - if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) { - zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1; - zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */ - zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize); - if (zbc->outBuff == NULL) return ERROR(memory_allocation); - } - - { size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize); - if (ZSTD_isError(errorCode)) return errorCode; } - - zbc->inToCompress = 0; - zbc->inBuffPos = 0; - zbc->inBuffTarget = zbc->blockSize; - zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_load; - zbc->checksum = params.fParams.checksumFlag > 0; - zbc->frameEnded = 0; - return 0; /* ready to go */ -} - - -size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel) -{ - ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); - return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0); -} - -size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel) -{ - return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel); -} - - -/* internal util function */ -MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - - -/* ====== Compression ====== */ - -typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e; - -static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr, - ZBUFF_flush_e const flush) -{ - U32 someMoreWork = 1; - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - - while (someMoreWork) { - switch(zbc->stage) - { - case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ - - case ZBUFFcs_load: - /* complete inBuffer */ - { size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos; - size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip); - zbc->inBuffPos += loaded; - ip += loaded; - if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) { - someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ - } } - /* compress current block (note : this stage cannot be stopped in the middle) */ - { void* cDst; - size_t cSize; - size_t const iSize = zbc->inBuffPos - zbc->inToCompress; - size_t oSize = oend-op; - if (oSize >= ZSTD_compressBound(iSize)) - cDst = op; /* compress directly into output buffer (avoid flush stage) */ - else - cDst = zbc->outBuff, oSize = zbc->outBuffSize; - cSize = (flush == zbf_end) ? - ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) : - ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize); - if (ZSTD_isError(cSize)) return cSize; - if (flush == zbf_end) zbc->frameEnded = 1; - /* prepare next block */ - zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize; - if (zbc->inBuffTarget > zbc->inBuffSize) - zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */ - zbc->inToCompress = zbc->inBuffPos; - if (cDst == op) { op += cSize; break; } /* no need to flush */ - zbc->outBuffContentSize = cSize; - zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_flush; /* continue to flush stage */ - } - - case ZBUFFcs_flush: - { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; - size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); - op += flushed; - zbc->outBuffFlushedSize += flushed; - if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ - zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0; - zbc->stage = ZBUFFcs_load; - break; - } - - case ZBUFFcs_final: - someMoreWork = 0; /* do nothing */ - break; - - default: - return ERROR(GENERIC); /* impossible */ - } - } - - *srcSizePtr = ip - istart; - *dstCapacityPtr = op - ostart; - if (zbc->frameEnded) return 0; - { size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos; - if (hintInSize==0) hintInSize = zbc->blockSize; - return hintInSize; - } -} - -size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather); -} - - - -/* ====== Finalize ====== */ - -size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) -{ - size_t srcSize = 0; - ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush); /* use a valid src address instead of NULL */ - return zbc->outBuffContentSize - zbc->outBuffFlushedSize; -} - - -size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr) -{ - BYTE* const ostart = (BYTE*)dst; - BYTE* const oend = ostart + *dstCapacityPtr; - BYTE* op = ostart; - - if (zbc->stage != ZBUFFcs_final) { - /* flush whatever remains */ - size_t outSize = *dstCapacityPtr; - size_t srcSize = 0; - size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end); /* use a valid address instead of NULL */ - size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; - op += outSize; - if (remainingToFlush) { - *dstCapacityPtr = op-ostart; - return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4); - } - /* create epilogue */ - zbc->stage = ZBUFFcs_final; - zbc->outBuffContentSize = !notEnded ? 0 : - ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* write epilogue into outBuff */ - } - - /* flush epilogue */ - { size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize; - size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush); - op += flushed; - zbc->outBuffFlushedSize += flushed; - *dstCapacityPtr = op-ostart; - if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */ - return toFlush - flushed; - } -} - - - -/* ************************************* -* Tool functions -***************************************/ -size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } -size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; } diff --git a/C/zstd/zbuff_decompress.c b/C/zstd/zbuff_decompress.c deleted file mode 100644 index 908120fc..00000000 --- a/C/zstd/zbuff_decompress.c +++ /dev/null @@ -1,297 +0,0 @@ -/* - Buffered version of Zstd compression library - Copyright (C) 2015-2016, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - zstd homepage : http://www.zstd.net/ -*/ - - -/* ************************************* -* Dependencies -***************************************/ -#include -#include "error_private.h" -#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */ -#define ZBUFF_STATIC_LINKING_ONLY -#include "zbuff.h" - - -/*-*************************************************************************** -* Streaming decompression howto -* -* A ZBUFF_DCtx object is required to track streaming operations. -* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources. -* Use ZBUFF_decompressInit() to start a new decompression operation, -* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary. -* Note that ZBUFF_DCtx objects can be re-init multiple times. -* -* Use ZBUFF_decompressContinue() repetitively to consume your input. -* *srcSizePtr and *dstCapacityPtr can be any size. -* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr. -* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again. -* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst. -* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency), -* or 0 when a frame is completely decoded, -* or an error code, which can be tested using ZBUFF_isError(). -* -* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize() -* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded. -* input : ZBUFF_recommendedDInSize == 128KB + 3; -* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 . -* *******************************************************************************/ - -typedef enum { ZBUFFds_init, ZBUFFds_loadHeader, - ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage; - -/* *** Resource management *** */ -struct ZBUFF_DCtx_s { - ZSTD_DCtx* zd; - ZSTD_frameParams fParams; - ZBUFF_dStage stage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; - size_t lhSize; - ZSTD_customMem customMem; -}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */ - - -ZBUFF_DCtx* ZBUFF_createDCtx(void) -{ - return ZBUFF_createDCtx_advanced(defaultCustomMem); -} - -ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem) -{ - ZBUFF_DCtx* zbd; - - if (!customMem.customAlloc && !customMem.customFree) - customMem = defaultCustomMem; - - if (!customMem.customAlloc || !customMem.customFree) - return NULL; - - zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx)); - if (zbd==NULL) return NULL; - memset(zbd, 0, sizeof(ZBUFF_DCtx)); - memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem)); - zbd->zd = ZSTD_createDCtx_advanced(customMem); - if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; } - zbd->stage = ZBUFFds_init; - return zbd; -} - -size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd) -{ - if (zbd==NULL) return 0; /* support free on null */ - ZSTD_freeDCtx(zbd->zd); - if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); - zbd->customMem.customFree(zbd->customMem.opaque, zbd); - return 0; -} - - -/* *** Initialization *** */ - -size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize) -{ - zbd->stage = ZBUFFds_loadHeader; - zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0; - return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize); -} - -size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd) -{ - return ZBUFF_decompressInitDictionary(zbd, NULL, 0); -} - - -/* internal util function */ -MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - size_t const length = MIN(dstCapacity, srcSize); - memcpy(dst, src, length); - return length; -} - - -/* *** Decompression *** */ - -size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd, - void* dst, size_t* dstCapacityPtr, - const void* src, size_t* srcSizePtr) -{ - const char* const istart = (const char*)src; - const char* const iend = istart + *srcSizePtr; - const char* ip = istart; - char* const ostart = (char*)dst; - char* const oend = ostart + *dstCapacityPtr; - char* op = ostart; - U32 someMoreWork = 1; - - while (someMoreWork) { - switch(zbd->stage) - { - case ZBUFFds_init : - return ERROR(init_missing); - - case ZBUFFds_loadHeader : - { size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize); - if (ZSTD_isError(hSize)) return hSize; - if (hSize != 0) { /* need more input */ - size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ - if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ - memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); - zbd->lhSize += iend-ip; - *dstCapacityPtr = 0; - return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ - } - memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad; - break; - } } - - /* Consume header */ - { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */ - size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size); - if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ - if (h1Size < zbd->lhSize) { /* long header */ - size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); - size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size); - if (ZSTD_isError(h2Result)) return h2Result; - } } - - zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); - - /* Frame header instruct buffer sizes */ - { size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); - size_t const neededOutSize = zbd->fParams.windowSize + blockSize; - zbd->blockSize = blockSize; - if (zbd->inBuffSize < blockSize) { - zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff); - zbd->inBuffSize = blockSize; - zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize); - if (zbd->inBuff == NULL) return ERROR(memory_allocation); - } - if (zbd->outBuffSize < neededOutSize) { - zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff); - zbd->outBuffSize = neededOutSize; - zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize); - if (zbd->outBuff == NULL) return ERROR(memory_allocation); - } } - zbd->stage = ZBUFFds_read; - /* pass-through */ - - case ZBUFFds_read: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (neededInSize==0) { /* end of frame */ - zbd->stage = ZBUFFds_init; - someMoreWork = 0; - break; - } - if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); - size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart), - ip, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - ip += neededInSize; - if (!decodedSize && !isSkipFrame) break; /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - break; - } - if (ip==iend) { someMoreWork = 0; break; } /* no more input */ - zbd->stage = ZBUFFds_load; - /* pass-through */ - } - - case ZBUFFds_load: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd); - size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */ - size_t loadedSize; - if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */ - loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip); - ip += loadedSize; - zbd->inPos += loadedSize; - if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ - - /* decode loaded input */ - { const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd); - size_t const decodedSize = ZSTD_decompressContinue(zbd->zd, - zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart, - zbd->inBuff, neededInSize); - if (ZSTD_isError(decodedSize)) return decodedSize; - zbd->inPos = 0; /* input is consumed */ - if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */ - zbd->outEnd = zbd->outStart + decodedSize; - zbd->stage = ZBUFFds_flush; - /* pass-through */ - } } - - case ZBUFFds_flush: - { size_t const toFlushSize = zbd->outEnd - zbd->outStart; - size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize); - op += flushedSize; - zbd->outStart += flushedSize; - if (flushedSize == toFlushSize) { /* flush completed */ - zbd->stage = ZBUFFds_read; - if (zbd->outStart + zbd->blockSize > zbd->outBuffSize) - zbd->outStart = zbd->outEnd = 0; - break; - } - /* cannot flush everything */ - someMoreWork = 0; - break; - } - default: return ERROR(GENERIC); /* impossible */ - } } - - /* result */ - *srcSizePtr = ip-istart; - *dstCapacityPtr = op-ostart; - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd); - if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */ - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block); - if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ - nextSrcSizeHint -= zbd->inPos; /* already loaded*/ - return nextSrcSizeHint; - } -} - - -/* ************************************* -* Tool functions -***************************************/ -size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; } -size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } diff --git a/C/zstd/zstd.h b/C/zstd/zstd.h index f6c5564d..f79a5dca 100644 --- a/C/zstd/zstd.h +++ b/C/zstd/zstd.h @@ -48,40 +48,41 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void); * Simple API ***************************************/ /*! ZSTD_compress() : - Compresses `src` buffer into already allocated `dst`. + Compresses `src` content as a single zstd compressed frame into already allocated `dst`. Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. - @return : the number of bytes written into `dst` (<= `dstCapacity), + @return : compressed size written into `dst` (<= `dstCapacity), or an error code if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); -/*! ZSTD_getDecompressedSize() : -* @return : decompressed size as a 64-bits value _if known_, 0 otherwise. -* note 1 : decompressed size can be very large (64-bits value), -* potentially larger than what local system can handle as a single memory segment. -* In which case, it's necessary to use streaming mode to decompress data. -* note 2 : decompressed size is an optional field, that may not be present. -* When `return==0`, data to decompress can have any size. -* In which case, it's necessary to use streaming mode to decompress data. -* Optionally, application may rely on its own implied limits. -* (For example, application data could be necessarily cut into blocks <= 16 KB). -* note 3 : decompressed size could be wrong or intentionally modified ! -* Always ensure result fits within application's authorized limits ! -* Each application can set its own limits. -* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); - /*! ZSTD_decompress() : - `compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. - `dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). - If originalSize is unknown, and if there is no implied application-specific limitations, - it's preferable to use streaming mode to decompress data. + `compressedSize` : must be the _exact_ size of a single compressed frame. + `dstCapacity` is an upper bound of originalSize. + If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), or an errorCode if it fails (which can be tested using ZSTD_isError()) */ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); +/*! ZSTD_getDecompressedSize() : +* 'src' is the start of a zstd compressed frame. +* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==0`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + /*====== Helper functions ======*/ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ @@ -200,7 +201,7 @@ typedef struct ZSTD_outBuffer_s { * Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. * * Use ZSTD_compressStream() repetitively to consume input stream. -* The function will automatically update both `pos`. +* The function will automatically update both `pos` fields. * Note that it may not consume the entire input, in which case `pos < size`, * and it's up to the caller to present again remaining data. * @return : a size hint, preferred nb of bytes to use as input for next function call @@ -229,7 +230,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); @@ -248,16 +249,17 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); * * Use ZSTD_initDStream() to start a new decompression operation, * or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* @return : recommended first input size * * Use ZSTD_decompressStream() repetitively to consume your input. -* The function will update both `pos`. -* Note that it may not consume the entire input (pos < size), -* in which case it's up to the caller to present remaining input again. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* If `output.pos < output.size`, decoder has flushed everything it could. * @return : 0 when a frame is completely decoded and fully flushed, -* 1 when there is still some data left within internal buffer to flush, -* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), -* or an error code, which can be tested using ZSTD_isError(). -* +* an error code, which can be tested using ZSTD_isError(), +* any other value > 0, which means there is still some work to do to complete the frame. +* The return value is a suggested next input size (just an hint, to help latency). * *******************************************************************************/ typedef struct ZSTD_DStream_s ZSTD_DStream; @@ -265,7 +267,7 @@ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ -ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); @@ -296,13 +298,14 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp #define ZSTD_HASHLOG3_MAX 17 #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) #define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_SEARCHLENGTH_MAX 7 -#define ZSTD_SEARCHLENGTH_MIN 3 +#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ #define ZSTD_TARGETLENGTH_MIN 4 #define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTD_frameHeaderSize_min = 5; +static const size_t ZSTD_frameHeaderSize_prefix = 5; +static const size_t ZSTD_frameHeaderSize_min = 6; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ @@ -349,14 +352,18 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); * Create a ZSTD compression context using external alloc and free functions */ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); + /*! ZSTD_createCDict_advanced() : * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem); -/*! ZSTD_sizeofCCtx() : - * Gives the amount of memory used by a given ZSTD_CCtx */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +/*! ZSTD_sizeof_CDict() : + * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); /*! ZSTD_getParams() : * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. @@ -396,10 +403,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * Create a ZSTD decompression context using external alloc and free functions */ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); -/*! ZSTD_sizeofDCtx() : +/*! ZSTD_sizeof_DCtx() : * Gives the amount of memory used by a given ZSTD_DCtx */ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +/*! ZSTD_sizeof_DDict() : + * Gives the amount of memory used by a given ZSTD_DDict */ +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + /* ****************************************************************** * Advanced Streaming functions @@ -410,7 +421,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, - ZSTD_parameters params, unsigned long long pledgedSrcSize); + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); @@ -421,6 +433,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); diff --git a/C/zstd/zstd_compress.c b/C/zstd/zstd_compress.c index 9e733b8f..f5b712fc 100644 --- a/C/zstd/zstd_compress.c +++ b/C/zstd/zstd_compress.c @@ -8,33 +8,13 @@ */ -/*-******************************************************* -* Compiler specifics -*********************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - /*-************************************* * Dependencies ***************************************/ #include /* memset */ #include "mem.h" #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ -#include "xxhash.h" /* XXH_reset, update, digest */ +#include "xxhash.h" /* XXH_reset, update, digest */ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" #define HUF_STATIC_LINKING_ONLY @@ -119,7 +99,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); if (!cctx) return NULL; memset(cctx, 0, sizeof(ZSTD_CCtx)); - memcpy(&(cctx->customMem), &customMem, sizeof(ZSTD_customMem)); + memcpy(&(cctx->customMem), &customMem, sizeof(customMem)); return cctx; } @@ -133,6 +113,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) { + if (cctx==NULL) return 0; /* support sizeof on NULL */ return sizeof(*cctx) + cctx->workSpaceSize; } @@ -142,19 +123,17 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface * } -#define CLAMP(val,min,max) { if (valmax) val=max; } -#define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); } - /** ZSTD_checkParams() : ensure param values remain within authorized range. @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { +# define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); } CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - { U32 const searchLengthMin = (cParams.strategy == ZSTD_fast || cParams.strategy == ZSTD_greedy) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; + { U32 const searchLengthMin = ((cParams.strategy == ZSTD_fast) | (cParams.strategy == ZSTD_greedy)) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); @@ -171,7 +150,7 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported); if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */ if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */ - if ((srcSize <= (1ULL << cParams.hashLog)) && ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */ + if ((srcSize <= (1ULL << cParams.hashLog)) & ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */ return ZSTD_checkCParams(cParams); } @@ -190,16 +169,16 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u { U32 const minSrcSize = (srcSize==0) ? 500 : 0; U64 const rSize = srcSize + dictSize + minSrcSize; if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; } } if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; - { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt); + { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt); U32 const maxChainLog = cPar.windowLog+btPlus; if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ - if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ + if ((cPar.hashLog < ZSTD_HASHLOG_MIN) & ((U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ return cPar; } @@ -226,81 +205,116 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) return sizeof(ZSTD_CCtx) + neededSpace; } + +static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2) +{ + return (param1.cParams.hashLog == param2.cParams.hashLog) + & (param1.cParams.chainLog == param2.cParams.chainLog) + & (param1.cParams.strategy == param2.cParams.strategy) + & ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3)); +} + +/*! ZSTD_continueCCtx() : + reuse CCtx without reset (note : requires no dictionary) */ +static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize) +{ + U32 const end = (U32)(cctx->nextSrc - cctx->base); + cctx->params = params; + cctx->frameContentSize = frameContentSize; + cctx->lowLimit = end; + cctx->dictLimit = end; + cctx->nextToUpdate = end+1; + cctx->stage = ZSTDcs_init; + cctx->dictID = 0; + cctx->loadedDictEnd = 0; + { int i; for (i=0; irep[i] = repStartValue[i]; } + cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ + XXH64_reset(&cctx->xxhState, 0); + return 0; +} + +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; + /*! ZSTD_resetCCtx_advanced() : - note : 'params' is expected to be validated */ + note : 'params' must be validated */ static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, ZSTD_parameters params, U64 frameContentSize, - U32 reset) -{ /* note : params considered validated here */ - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); - U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; - size_t const tokenSpace = blockSize + 11*maxNbSeq; - size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); - size_t const hSize = ((size_t)1) << params.cParams.hashLog; - U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); - size_t const h3Size = ((size_t)1) << hashLog3; - size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); - void* ptr; + ZSTD_compResetPolicy_e const crp) +{ + if (crp == ZSTDcrp_continue) + if (ZSTD_equivalentParams(params, zc->params)) + return ZSTD_continueCCtx(zc, params, frameContentSize); - /* Check if workSpace is large enough, alloc a new one if needed */ - { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) { - ZSTD_free(zc->workSpace, zc->customMem); - zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); - if (zc->workSpace == NULL) return ERROR(memory_allocation); - zc->workSpaceSize = neededSpace; - } } + { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); + U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); + size_t const hSize = ((size_t)1) << params.cParams.hashLog; + U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + void* ptr; - if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ - XXH64_reset(&zc->xxhState, 0); - zc->hashLog3 = hashLog3; - zc->hashTable = (U32*)(zc->workSpace); - zc->chainTable = zc->hashTable + hSize; - zc->hashTable3 = zc->chainTable + chainSize; - ptr = zc->hashTable3 + h3Size; - zc->hufTable = (HUF_CElt*)ptr; - zc->flagStaticTables = 0; - ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ + /* Check if workSpace is large enough, alloc a new one if needed */ + { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) { + ZSTD_free(zc->workSpace, zc->customMem); + zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); + if (zc->workSpace == NULL) return ERROR(memory_allocation); + zc->workSpaceSize = neededSpace; + } } - zc->nextToUpdate = 1; - zc->nextSrc = NULL; - zc->base = NULL; - zc->dictBase = NULL; - zc->dictLimit = 0; - zc->lowLimit = 0; - zc->params = params; - zc->blockSize = blockSize; - zc->frameContentSize = frameContentSize; - { int i; for (i=0; irep[i] = repStartValue[i]; } + if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */ + XXH64_reset(&zc->xxhState, 0); + zc->hashLog3 = hashLog3; + zc->hashTable = (U32*)(zc->workSpace); + zc->chainTable = zc->hashTable + hSize; + zc->hashTable3 = zc->chainTable + chainSize; + ptr = zc->hashTable3 + h3Size; + zc->hufTable = (HUF_CElt*)ptr; + zc->flagStaticTables = 0; + ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ - if (params.cParams.strategy == ZSTD_btopt) { - zc->seqStore.litFreq = (U32*)ptr; - zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); - zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); - ptr = zc->seqStore.offCodeFreq + (MaxOff+1); - zc->seqStore.matchTable = (ZSTD_match_t*)ptr; - ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; - zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; - ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; - zc->seqStore.litLengthSum = 0; + zc->nextToUpdate = 1; + zc->nextSrc = NULL; + zc->base = NULL; + zc->dictBase = NULL; + zc->dictLimit = 0; + zc->lowLimit = 0; + zc->params = params; + zc->blockSize = blockSize; + zc->frameContentSize = frameContentSize; + { int i; for (i=0; irep[i] = repStartValue[i]; } + + if (params.cParams.strategy == ZSTD_btopt) { + zc->seqStore.litFreq = (U32*)ptr; + zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); + zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); + ptr = zc->seqStore.offCodeFreq + (MaxOff+1); + zc->seqStore.matchTable = (ZSTD_match_t*)ptr; + ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; + zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; + ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; + zc->seqStore.litLengthSum = 0; + } + zc->seqStore.sequencesStart = (seqDef*)ptr; + ptr = zc->seqStore.sequencesStart + maxNbSeq; + zc->seqStore.llCode = (BYTE*) ptr; + zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; + zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; + zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; + + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->loadedDictEnd = 0; + + return 0; } - zc->seqStore.sequencesStart = (seqDef*)ptr; - ptr = zc->seqStore.sequencesStart + maxNbSeq; - zc->seqStore.llCode = (BYTE*) ptr; - zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; - zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; - zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; - - zc->stage = ZSTDcs_init; - zc->dictID = 0; - zc->loadedDictEnd = 0; - - return 0; } @@ -313,7 +327,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx) if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); - ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, ZSTDcrp_noMemset); dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ /* copy tables */ @@ -668,8 +682,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc, FSE_CState_t stateOffsetBits; FSE_CState_t stateLitLength; - { size_t const errorCode = BIT_initCStream(&blockStream, op, oend-op); - if (ERR_isError(errorCode)) return ERROR(dstSize_tooSmall); } /* not enough space remaining */ + CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */ /* first symbols */ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); @@ -1083,7 +1096,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, if (ip <= ilimit) { /* Fill Table */ - hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); /* check immediate repcode */ while (ip <= ilimit) { @@ -2476,8 +2489,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -2485,8 +2497,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -2494,8 +2505,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_ unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } @@ -2520,9 +2530,9 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); /* known magic number : dict is parsed for entropy stats and content */ - { size_t const eSize_8 = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); - size_t const eSize = eSize_8 + 8; - if (ZSTD_isError(eSize_8)) return eSize_8; + { size_t const loadError = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); + size_t const eSize = loadError + 8; + if (ZSTD_isError(loadError)) return loadError; return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); } } @@ -2530,14 +2540,13 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si /*! ZSTD_compressBegin_internal() : * @return : 0, or an error code */ -static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, U64 pledgedSrcSize) { - size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); - if (ZSTD_isError(resetError)) return resetError; - - return ZSTD_compress_insertDictionary(zc, dict, dictSize); + ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; + CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); + return ZSTD_compress_insertDictionary(cctx, dict, dictSize); } @@ -2548,9 +2557,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, ZSTD_parameters params, unsigned long long pledgedSrcSize) { /* compression parameters verification and optimization */ - { size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); - if (ZSTD_isError(errorCode)) return errorCode; } - + CHECK_F(ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize)); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); } @@ -2628,9 +2635,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize); - if(ZSTD_isError(errorCode)) return errorCode; - + CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); } @@ -2640,8 +2645,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, const void* dict,size_t dictSize, ZSTD_parameters params) { - size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, srcSize); - if (ZSTD_isError(errorCode)) return errorCode; + CHECK_F(ZSTD_checkCParams_advanced(params.cParams, srcSize)); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); } @@ -2677,6 +2681,12 @@ struct ZSTD_CDict_s { ZSTD_CCtx* refContext; }; /* typedef'd tp ZSTD_CDict within "zstd.h" */ +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; +} + ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) { if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; @@ -2728,17 +2738,23 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict) } } +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) +{ + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext)) + else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); + return 0; +} + /*! ZSTD_compress_usingCDict() : * Compression using a digested Dictionary. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Note that compression level is decided during dictionary creation */ -ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict) +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) { - size_t const errorCode = ZSTD_copyCCtx(cctx, cdict->refContext); - if (ZSTD_isError(errorCode)) return errorCode; + CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); if (cdict->refContext->params.fParams.contentSizeFlag==1) { cctx->params.fParams.contentSizeFlag = 1; @@ -2757,7 +2773,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; struct ZSTD_CStream_s { - ZSTD_CCtx* zc; + ZSTD_CCtx* cctx; + ZSTD_CDict* cdict; char* inBuff; size_t inBuffSize; size_t inToCompress; @@ -2790,8 +2807,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) if (zcs==NULL) return NULL; memset(zcs, 0, sizeof(ZSTD_CStream)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); - zcs->zc = ZSTD_createCCtx_advanced(customMem); - if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; } + zcs->cctx = ZSTD_createCCtx_advanced(customMem); + if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } return zcs; } @@ -2799,7 +2816,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) { if (zcs==NULL) return 0; /* support free on NULL */ { ZSTD_customMem const cMem = zcs->customMem; - ZSTD_freeCCtx(zcs->zc); + ZSTD_freeCCtx(zcs->cctx); + ZSTD_freeCDict(zcs->cdict); ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs, cMem); @@ -2813,6 +2831,19 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs) size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)); + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->frameEnded = 0; + return 0; /* ready to go */ +} + size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize) @@ -2834,17 +2865,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, if (zcs->outBuff == NULL) return ERROR(memory_allocation); } - { size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize); - if (ZSTD_isError(errorCode)) return errorCode; } + ZSTD_freeCDict(zcs->cdict); + zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + if (zcs->cdict == NULL) return ERROR(memory_allocation); - zcs->inToCompress = 0; - zcs->inBuffPos = 0; - zcs->inBuffTarget = zcs->blockSize; - zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; - zcs->stage = zcss_load; zcs->checksum = params.fParams.checksumFlag > 0; - zcs->frameEnded = 0; - return 0; /* ready to go */ + + return ZSTD_resetCStream(zcs, pledgedSrcSize); } size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) @@ -2860,7 +2887,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) { - return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; + if (zcs==NULL) return 0; /* support sizeof on NULL */ + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize; } /*====== Compression ======*/ @@ -2911,8 +2939,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, else cDst = zcs->outBuff, oSize = zcs->outBuffSize; cSize = (flush == zsf_end) ? - ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); if (ZSTD_isError(cSize)) return cSize; if (flush == zsf_end) zcs->frameEnded = 1; /* prepare next block */ @@ -2977,8 +3005,8 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) size_t srcSize = 0; size_t sizeWritten = output->size - output->pos; size_t const result = ZSTD_compressStream_generic(zcs, - (char*)(output->dst) + output->pos, &sizeWritten, - &srcSize, &srcSize, /* use a valid src address instead of NULL */ + (char*)(output->dst) + output->pos, &sizeWritten, + &srcSize, &srcSize, /* use a valid src address instead of NULL */ zsf_flush); output->pos += sizeWritten; if (ZSTD_isError(result)) return result; @@ -3006,7 +3034,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) /* create epilogue */ zcs->stage = zcss_final; zcs->outBuffContentSize = !notEnded ? 0 : - ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ } /* flush epilogue */ diff --git a/C/zstd/zstd_decompress.c b/C/zstd/zstd_decompress.c index 762e972c..2b2539a4 100644 --- a/C/zstd/zstd_decompress.c +++ b/C/zstd/zstd_decompress.c @@ -28,7 +28,6 @@ # define ZSTD_LEGACY_SUPPORT 0 #endif - /*! * MAXWINDOWSIZE_DEFAULT : * maximum window size accepted by DStream, by default. @@ -57,28 +56,6 @@ #endif -/*-******************************************************* -* Compiler specifics -*********************************************************/ -#ifdef _MSC_VER /* Visual Studio */ -# define FORCE_INLINE static __forceinline -# include /* For Visual 2005 */ -# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ -# pragma warning(disable : 4324) /* disable: C4324: padded structure */ -# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ -#else -# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -# else -# define FORCE_INLINE static -# endif /* __STDC_VERSION__ */ -#endif - - /*-************************************* * Macros ***************************************/ @@ -103,8 +80,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, struct ZSTD_DCtx_s { + const FSE_DTable* LLTptr; + const FSE_DTable* MLTptr; + const FSE_DTable* OFTptr; + const HUF_DTable* HUFptr; FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; - FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ const void* previousDstEnd; @@ -130,23 +111,27 @@ struct ZSTD_DCtx_s BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; }; /* typedef'd to ZSTD_DCtx within "zstd.h" */ -size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) { - dctx->expected = ZSTD_frameHeaderSize_min; + dctx->expected = ZSTD_frameHeaderSize_prefix; dctx->stage = ZSTDds_getFrameHeaderSize; dctx->previousDstEnd = NULL; dctx->base = NULL; dctx->vBase = NULL; dctx->dictEnd = NULL; - dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); + dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; - MEM_STATIC_ASSERT(sizeof(dctx->rep)==sizeof(repStartValue)); - memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); + MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue)); + memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->LLTable; + dctx->MLTptr = dctx->MLTable; + dctx->OFTptr = dctx->OFTable; + dctx->HUFptr = dctx->hufTable; return 0; } @@ -157,7 +142,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc || !customMem.customFree) return NULL; - dctx = (ZSTD_DCtx*) ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); + dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); if (!dctx) return NULL; memcpy(&dctx->customMem, &customMem, sizeof(customMem)); ZSTD_decompressBegin(dctx); @@ -182,6 +167,25 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ } +static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + ZSTD_decompressBegin(dstDCtx); /* init */ + dstDCtx->dictEnd = srcDCtx->dictEnd; + dstDCtx->vBase = srcDCtx->vBase; + dstDCtx->base = srcDCtx->base; + dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; + dstDCtx->dictID = srcDCtx->dictID; + dstDCtx->litEntropy = srcDCtx->litEntropy; + dstDCtx->fseEntropy = srcDCtx->fseEntropy; + dstDCtx->LLTptr = srcDCtx->LLTable; + dstDCtx->MLTptr = srcDCtx->MLTable; + dstDCtx->OFTptr = srcDCtx->OFTable; + dstDCtx->HUFptr = srcDCtx->hufTable; + dstDCtx->rep[0] = srcDCtx->rep[0]; + dstDCtx->rep[1] = srcDCtx->rep[1]; + dstDCtx->rep[2] = srcDCtx->rep[2]; +} + /*-************************************************************* * Decompression section @@ -190,16 +194,16 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) /* See compression format details in : zstd_compression_format.md */ /** ZSTD_frameHeaderSize() : -* srcSize must be >= ZSTD_frameHeaderSize_min. +* srcSize must be >= ZSTD_frameHeaderSize_prefix. * @return : size of the Frame Header */ static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) { - if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); + if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); { BYTE const fhd = ((const BYTE*)src)[4]; U32 const dictID= fhd & 3; U32 const singleSegment = (fhd >> 5) & 1; U32 const fcsId = fhd >> 6; - return ZSTD_frameHeaderSize_min + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + (singleSegment && !fcsId); } } @@ -214,7 +218,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t { const BYTE* ip = (const BYTE*)src; - if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; + if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix; if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ @@ -373,34 +377,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ /* 2 - 2 - 10 - 10 */ - { singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - } + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; case 2: /* 2 - 2 - 14 - 14 */ - { lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - } + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; case 3: /* 2 - 2 - 18 - 18 */ - { lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; } if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (HUF_isError((litEncType==set_repeat) ? ( singleStream ? - HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) : - HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) : + HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : + HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : ( singleStream ? HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) @@ -410,6 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litSize = litSize; dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; return litCSize + lhSize; } @@ -476,7 +478,6 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, default: return ERROR(corruption_detected); /* impossible */ } - } } @@ -485,7 +486,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, @return : nb bytes read from src, or an error code if it fails, testable with ZSTD_isError() */ -FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, +static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, const void* src, size_t srcSize, const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) { @@ -515,8 +516,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e } -size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, - FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable, +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, const void* src, size_t srcSize) { const BYTE* const istart = (const BYTE* const)src; @@ -546,16 +546,19 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, ip++; /* Build DTables */ - { size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); + { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + if (LLtype != set_repeat) dctx->LLTptr = dctx->LLTable; ip += llhSize; } - { size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + if (OFtype != set_repeat) dctx->OFTptr = dctx->OFTable; ip += ofhSize; } - { size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + if (MLtype != set_repeat) dctx->MLTptr = dctx->MLTable; ip += mlhSize; } } @@ -738,16 +741,13 @@ static size_t ZSTD_decompressSequences( const BYTE* litPtr = dctx->litPtr; const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; const BYTE* const litEnd = litPtr + dctx->litSize; - FSE_DTable* DTableLL = dctx->LLTable; - FSE_DTable* DTableML = dctx->MLTable; - FSE_DTable* DTableOffb = dctx->OffTable; const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); int nbSeq; /* Build Decoding Tables */ - { size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); if (ZSTD_isError(seqHSize)) return seqHSize; ip += seqHSize; } @@ -757,11 +757,10 @@ static size_t ZSTD_decompressSequences( seqState_t seqState; dctx->fseEntropy = 1; { U32 i; for (i=0; irep[i]; } - { size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); - if (ERR_isError(errorCode)) return ERROR(corruption_detected); } - FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); - FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); - FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML); + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { nbSeq--; @@ -863,12 +862,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); /* Frame Header */ - { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); - size_t result; + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); - result = ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize); - if (ZSTD_isError(result)) return result; + CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize)); ip += frameHeaderSize; remainingSize -= frameHeaderSize; } @@ -921,25 +918,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } -/*! ZSTD_decompress_usingPreparedDCtx() : -* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded. -* It avoids reloading the dictionary each time. -* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict(). -* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */ -size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - ZSTD_copyDCtx(dctx, refDCtx); - ZSTD_checkContinuity(dctx, dst); - return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); -} - - size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict, size_t dictSize) + const void* src, size_t srcSize, + const void* dict, size_t dictSize) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); @@ -1013,32 +995,30 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c switch (dctx->stage) { case ZSTDds_getFrameHeaderSize : - if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */ + if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ dctx->stage = ZSTDds_decodeSkippableHeader; return 0; } - dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; - memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); - if (dctx->headerSize > ZSTD_frameHeaderSize_min) { - dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); + if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { + dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; dctx->stage = ZSTDds_decodeFrameHeader; return 0; } dctx->expected = 0; /* not necessary to copy more */ case ZSTDds_decodeFrameHeader: - { size_t result; - memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); - result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); - if (ZSTD_isError(result)) return result; - dctx->expected = ZSTD_blockHeaderSize; - dctx->stage = ZSTDds_decodeBlockHeader; - return 0; - } + memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); + CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + case ZSTDds_decodeBlockHeader: { blockProperties_t bp; size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); @@ -1110,7 +1090,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c return 0; } case ZSTDds_decodeSkippableHeader: - { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); dctx->stage = ZSTDds_skipFrame; return 0; @@ -1149,8 +1129,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -1158,8 +1137,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -1167,8 +1145,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); - { size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); - if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); } + CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); dictPtr += litlengthHeaderSize; } @@ -1204,21 +1181,16 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict return ZSTD_refDictContent(dctx, dict, dictSize); } - size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) { - { size_t const errorCode = ZSTD_decompressBegin(dctx); - if (ZSTD_isError(errorCode)) return errorCode; } - - if (dict && dictSize) { - size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize); - if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted); - } - + CHECK_F(ZSTD_decompressBegin(dctx)); + if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); return 0; } +/* ====== ZSTD_DDict ====== */ + struct ZSTD_DDict_s { void* dict; size_t dictSize; @@ -1277,20 +1249,27 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict) } } +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; +} + + /*! ZSTD_decompress_usingDDict() : * Decompression using a pre-digested Dictionary * Use dictionary without significant overhead. */ -ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_DDict* ddict) +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); #endif - return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, - dst, dstCapacity, - src, srcSize); + ZSTD_refDCtx(dctx, ddict->refContext); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); } @@ -1303,7 +1282,8 @@ typedef enum { zdss_init, zdss_loadHeader, /* *** Resource management *** */ struct ZSTD_DStream_s { - ZSTD_DCtx* zd; + ZSTD_DCtx* dctx; + ZSTD_DDict* ddict; ZSTD_frameParams fParams; ZSTD_dStreamStage stage; char* inBuff; @@ -1315,15 +1295,13 @@ struct ZSTD_DStream_s { size_t outStart; size_t outEnd; size_t blockSize; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ size_t lhSize; ZSTD_customMem customMem; - void* dictContent; - size_t dictSize; - const void* dictSource; void* legacyContext; U32 previousLegacyVersion; U32 legacyVersion; + U32 hostageByte; }; /* typedef'd to ZSTD_DStream within "zstd.h" */ @@ -1343,8 +1321,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) if (zds==NULL) return NULL; memset(zds, 0, sizeof(ZSTD_DStream)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); - zds->zd = ZSTD_createDCtx_advanced(customMem); - if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->dctx = ZSTD_createDCtx_advanced(customMem); + if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } zds->stage = zdss_init; zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; return zds; @@ -1354,10 +1332,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds) { if (zds==NULL) return 0; /* support free on null */ { ZSTD_customMem const cMem = zds->customMem; - ZSTD_freeDCtx(zds->zd); + ZSTD_freeDCtx(zds->dctx); + ZSTD_freeDDict(zds->ddict); ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->outBuff, cMem); - ZSTD_free(zds->dictContent, cMem); #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (zds->legacyContext) ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); @@ -1377,17 +1355,12 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di { zds->stage = zdss_loadHeader; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; - if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */ - if (dictSize > zds->dictSize) { - ZSTD_free(zds->dictContent, zds->customMem); - zds->dictContent = ZSTD_malloc(dictSize, zds->customMem); - if (zds->dictContent == NULL) return ERROR(memory_allocation); - } - memcpy(zds->dictContent, dict, dictSize); - zds->dictSize = dictSize; - } + ZSTD_freeDDict(zds->ddict); + zds->ddict = ZSTD_createDDict(dict, dictSize); + if (zds->ddict == NULL) return ERROR(memory_allocation); zds->legacyVersion = 0; - return 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; } size_t ZSTD_initDStream(ZSTD_DStream* zds) @@ -1395,6 +1368,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) return ZSTD_initDStream_usingDict(zds, NULL, 0); } +size_t ZSTD_resetDStream(ZSTD_DStream* zds) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue) { @@ -1409,11 +1391,12 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) { - return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; + if (zds==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize; } -/* *** Decompression *** */ +/* ***** Decompression ***** */ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) { @@ -1450,10 +1433,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { - size_t initResult; - initResult = ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, - zds->dictContent, zds->dictSize); - if (ZSTD_isError(initResult)) return initResult; + CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, + zds->ddict->dict, zds->ddict->dictSize)); zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); } else { @@ -1468,27 +1449,24 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); zds->lhSize += iend-ip; input->pos = input->size; - return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; break; } } /* Consume header */ - ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize); - { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ - size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); - if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ - if (h1Size < zds->lhSize) { /* long header */ - size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd); - size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size); - if (ZSTD_isError(h2Result)) return h2Result; + ZSTD_refDCtx(zds->dctx, zds->ddict->refContext); + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); + { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); } } zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); - /* Frame header instruct buffer sizes */ + /* Adapt buffer sizes to frame header instructions */ { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); size_t const neededOutSize = zds->fParams.windowSize + blockSize; zds->blockSize = blockSize; @@ -1508,21 +1486,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* pass-through */ case zdss_read: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); if (neededInSize==0) { /* end of frame */ zds->stage = zdss_init; someMoreWork = 0; break; } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ - const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); - size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), ip, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; ip += neededInSize; if (!decodedSize && !isSkipFrame) break; /* this was just a header */ - zds->outEnd = zds->outStart + decodedSize; + zds->outEnd = zds->outStart + decodedSize; zds->stage = zdss_flush; break; } @@ -1532,7 +1510,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } case zdss_load: - { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ size_t loadedSize; if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ @@ -1542,8 +1520,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ - { const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); - size_t const decodedSize = ZSTD_decompressContinue(zds->zd, + { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, zds->inBuff, neededInSize); if (ZSTD_isError(decodedSize)) return decodedSize; @@ -1565,7 +1543,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->outStart = zds->outEnd = 0; break; } - /* cannot flush everything */ + /* cannot complete flush */ someMoreWork = 0; break; } @@ -1575,9 +1553,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB /* result */ input->pos += (size_t)(ip-istart); output->pos += (size_t)(op-ostart); - { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); - if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ - nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */ + input->pos++; /* release hostage */ + } + return 0; + } + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ nextSrcSizeHint -= zds->inPos; /* already loaded*/ return nextSrcSizeHint; diff --git a/C/zstd/zstd_internal.h b/C/zstd/zstd_internal.h index 51e7170e..987d9386 100644 --- a/C/zstd/zstd_internal.h +++ b/C/zstd/zstd_internal.h @@ -10,6 +10,28 @@ #ifndef ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE +/*-******************************************************* +* Compiler specifics +*********************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + /*-************************************* * Dependencies ***************************************/ @@ -20,10 +42,12 @@ /*-************************************* -* Common macros +* shared macros ***************************************/ #define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ +#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ /*-************************************* diff --git a/C/zstd/zstd_v05.c b/C/zstd/zstd_v05.c index 8434f613..5027e2b8 100644 --- a/C/zstd/zstd_v05.c +++ b/C/zstd/zstd_v05.c @@ -10,6 +10,7 @@ /*- Dependencies -*/ #include "zstd_v05.h" +#include "error_private.h" /* ****************************************************************** @@ -254,79 +255,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) #endif /* MEM_H_MODULE */ -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/zstd -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - ZSTDv05_error_no_error, - ZSTDv05_error_GENERIC, - ZSTDv05_error_prefix_unknown, - ZSTDv05_error_frameParameter_unsupported, - ZSTDv05_error_frameParameter_unsupportedBy32bits, - ZSTDv05_error_init_missing, - ZSTDv05_error_memory_allocation, - ZSTDv05_error_stage_wrong, - ZSTDv05_error_dstSize_tooSmall, - ZSTDv05_error_srcSize_wrong, - ZSTDv05_error_corruption_detected, - ZSTDv05_error_tableLog_tooLarge, - ZSTDv05_error_maxSymbolValue_tooLarge, - ZSTDv05_error_maxSymbolValue_tooSmall, - ZSTDv05_error_dictionary_corrupted, - ZSTDv05_error_maxCode -} ZSTDv05_ErrorCode; - -/* note : functions provide error codes in reverse negative order, - so compare with (size_t)(0-enum) */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ - - /* zstd - standard compression library Header File for static linking only @@ -470,119 +398,6 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity #endif /* ZSTDv05_STATIC_H */ - -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Source repository : https://github.com/Cyan4973/zstd -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization -******************************************/ -typedef ZSTDv05_ErrorCode ERR_enum; -#define PREFIX(name) ZSTDv05_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) (size_t)-PREFIX(name) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getError(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( ERR_getError(code) ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max possible Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(maxCode): - default: return notErrorCode; /* should be impossible, due to ERR_getError() */ - } -} - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ /* zstd_internal - common functions to include Header File for include @@ -2883,9 +2698,6 @@ static void ZSTDv05_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } * tells if a return value is an error code */ unsigned ZSTDv05_isError(size_t code) { return ERR_isError(code); } -/*! ZSTDv05_getError() : -* convert a `size_t` function result into a proper ZSTDv05_errorCode enum */ -ZSTDv05_ErrorCode ZSTDv05_getError(size_t code) { return ERR_getError(code); } /*! ZSTDv05_getErrorName() : * provides error code string (useful for debugging) */ diff --git a/C/zstd/zstd_v06.c b/C/zstd/zstd_v06.c index 88236148..5a9bc40e 100644 --- a/C/zstd/zstd_v06.c +++ b/C/zstd/zstd_v06.c @@ -13,6 +13,7 @@ #include /* size_t, ptrdiff_t */ #include /* memcpy */ #include /* malloc, free, qsort */ +#include "error_private.h" @@ -273,77 +274,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) #endif /* MEM_H_MODULE */ -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - ZSTDv06_error_no_error, - ZSTDv06_error_GENERIC, - ZSTDv06_error_prefix_unknown, - ZSTDv06_error_frameParameter_unsupported, - ZSTDv06_error_frameParameter_unsupportedBy32bits, - ZSTDv06_error_compressionParameter_unsupported, - ZSTDv06_error_init_missing, - ZSTDv06_error_memory_allocation, - ZSTDv06_error_stage_wrong, - ZSTDv06_error_dstSize_tooSmall, - ZSTDv06_error_srcSize_wrong, - ZSTDv06_error_corruption_detected, - ZSTDv06_error_tableLog_tooLarge, - ZSTDv06_error_maxSymbolValue_tooLarge, - ZSTDv06_error_maxSymbolValue_tooSmall, - ZSTDv06_error_dictionary_corrupted, - ZSTDv06_error_maxCode -} ZSTDv06_ErrorCode; - -/* note : compare with size_t function results using ZSTDv06_getError() */ - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ /* zstd - standard compression library Header File for static linking only @@ -469,137 +399,12 @@ ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx); ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/*-************************************* -* Error management -***************************************/ -/*! ZSTDv06_getErrorCode() : - convert a `size_t` function result into a `ZSTDv06_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDLIB_API ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t functionResult); -ZSTDLIB_API const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code); - #if defined (__cplusplus) } #endif #endif /* ZSTDv06_STATIC_H */ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTDv06_ErrorCode ERR_enum; -#define PREFIX(name) ZSTDv06_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ /* zstd_internal - common functions to include Header File for include @@ -2986,14 +2791,6 @@ unsigned ZSTDv06_isError(size_t code) { return ERR_isError(code); } * provides error code string from function result (useful for debugging) */ const char* ZSTDv06_getErrorName(size_t code) { return ERR_getErrorName(code); } -/*! ZSTDv06_getError() : -* convert a `size_t` function result into a proper ZSTDv06_errorCode enum */ -ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTDv06_getErrorString() : -* provides error code string from enum */ -const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code) { return ERR_getErrorName(code); } - /* ************************************************************** * ZBUFF Error Management diff --git a/C/zstd/zstd_v07.c b/C/zstd/zstd_v07.c index b634d624..dac71aeb 100644 --- a/C/zstd/zstd_v07.c +++ b/C/zstd/zstd_v07.c @@ -21,6 +21,8 @@ #define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ #define ZSTDv07_STATIC_LINKING_ONLY +#include "error_private.h" + #ifdef ZSTDv07_STATIC_LINKING_ONLY @@ -427,203 +429,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) #endif #endif /* MEM_H_MODULE */ - -/* ****************************************************************** - Error codes list - Copyright (C) 2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -#ifndef ERROR_PUBLIC_H_MODULE -#define ERROR_PUBLIC_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - -/* **************************************** -* error codes list -******************************************/ -typedef enum { - ZSTDv07_error_no_error, - ZSTDv07_error_GENERIC, - ZSTDv07_error_prefix_unknown, - ZSTDv07_error_frameParameter_unsupported, - ZSTDv07_error_frameParameter_unsupportedBy32bits, - ZSTDv07_error_compressionParameter_unsupported, - ZSTDv07_error_init_missing, - ZSTDv07_error_memory_allocation, - ZSTDv07_error_stage_wrong, - ZSTDv07_error_dstSize_tooSmall, - ZSTDv07_error_srcSize_wrong, - ZSTDv07_error_corruption_detected, - ZSTDv07_error_checksum_wrong, - ZSTDv07_error_tableLog_tooLarge, - ZSTDv07_error_maxSymbolValue_tooLarge, - ZSTDv07_error_maxSymbolValue_tooSmall, - ZSTDv07_error_dictionary_corrupted, - ZSTDv07_error_dictionary_wrong, - ZSTDv07_error_maxCode -} ZSTDv07_ErrorCode; - -/*! ZSTDv07_getErrorCode() : - convert a `size_t` function result into a `ZSTDv07_ErrorCode` enum type, - which can be used to compare directly with enum list published into "error_public.h" */ -ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t functionResult); -const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code); - - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_PUBLIC_H_MODULE */ -/* ****************************************************************** - Error codes and messages - Copyright (C) 2013-2016, Yann Collet - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - Homepage : http://www.zstd.net -****************************************************************** */ -/* Note : this module is expected to remain private, do not expose it */ - -#ifndef ERROR_H_MODULE -#define ERROR_H_MODULE - -#if defined (__cplusplus) -extern "C" { -#endif - - - -/* **************************************** -* Compiler-specific -******************************************/ -#if defined(__GNUC__) -# define ERR_STATIC static __attribute__((unused)) -#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) -# define ERR_STATIC static inline -#elif defined(_MSC_VER) -# define ERR_STATIC static __inline -#else -# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ -#endif - - -/*-**************************************** -* Customization (error_public.h) -******************************************/ -typedef ZSTDv07_ErrorCode ERR_enum; -#define PREFIX(name) ZSTDv07_error_##name - - -/*-**************************************** -* Error codes handling -******************************************/ -#ifdef ERROR -# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ -#endif -#define ERROR(name) ((size_t)-PREFIX(name)) - -ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } - -ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } - - -/*-**************************************** -* Error Strings -******************************************/ - -ERR_STATIC const char* ERR_getErrorString(ERR_enum code) -{ - static const char* notErrorCode = "Unspecified error code"; - switch( code ) - { - case PREFIX(no_error): return "No error detected"; - case PREFIX(GENERIC): return "Error (generic)"; - case PREFIX(prefix_unknown): return "Unknown frame descriptor"; - case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; - case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; - case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; - case PREFIX(init_missing): return "Context should be init first"; - case PREFIX(memory_allocation): return "Allocation error : not enough memory"; - case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; - case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; - case PREFIX(srcSize_wrong): return "Src size incorrect"; - case PREFIX(corruption_detected): return "Corrupted block detected"; - case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; - case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; - case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; - case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; - case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; - case PREFIX(dictionary_wrong): return "Dictionary mismatch"; - case PREFIX(maxCode): - default: return notErrorCode; - } -} - -ERR_STATIC const char* ERR_getErrorName(size_t code) -{ - return ERR_getErrorString(ERR_getErrorCode(code)); -} - -#if defined (__cplusplus) -} -#endif - -#endif /* ERROR_H_MODULE */ /* ****************************************************************** bitstream Part of FSE library @@ -2882,13 +2687,6 @@ unsigned ZSTDv07_isError(size_t code) { return ERR_isError(code); } * provides error code string from function result (useful for debugging) */ const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); } -/*! ZSTDv07_getError() : -* convert a `size_t` function result into a proper ZSTDv07_errorCode enum */ -ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t code) { return ERR_getErrorCode(code); } - -/*! ZSTDv07_getErrorString() : -* provides error code string from enum */ -const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code) { return ERR_getErrorName(code); } /* ************************************************************** diff --git a/CPP/7zip/Archive/Icons/zst.ico b/CPP/7zip/Archive/Icons/zst.ico new file mode 100644 index 00000000..e69de29b diff --git a/CPP/7zip/Archive/ZstdHandler.cpp b/CPP/7zip/Archive/ZstdHandler.cpp new file mode 100644 index 00000000..9913a6ff --- /dev/null +++ b/CPP/7zip/Archive/ZstdHandler.cpp @@ -0,0 +1,367 @@ +// ZstdHandler.cpp + +#include "StdAfx.h" + +#include "../../../C/CpuArch.h" +#include "../../Common/ComTry.h" +#include "../../Common/Defs.h" + +#include "../Common/ProgressUtils.h" +#include "../Common/RegisterArc.h" +#include "../Common/StreamUtils.h" + +#include "../Compress/ZstdDecoder.h" +#include "../Compress/ZstdEncoder.h" +#include "../Compress/CopyCoder.h" + +#include "Common/DummyOutStream.h" +#include "Common/HandlerOut.h" + +#include + +using namespace NWindows; + +namespace NArchive { +namespace NZSTD { + +class CHandler: + public IInArchive, + public IArchiveOpenSeq, + public IOutArchive, + public ISetProperties, + public CMyUnknownImp +{ + CMyComPtr _stream; + CMyComPtr _seqStream; + + bool _isArc; + bool _dataAfterEnd; + bool _needMoreInput; + + bool _packSize_Defined; + bool _unpackSize_Defined; + + UInt64 _packSize; + UInt64 _unpackSize; + UInt64 _numStreams; + UInt64 _numBlocks; + + CSingleMethodProps _props; + +public: + MY_UNKNOWN_IMP4( + IInArchive, + IArchiveOpenSeq, + IOutArchive, + ISetProperties) + INTERFACE_IInArchive(;) + INTERFACE_IOutArchive(;) + STDMETHOD(OpenSeq)(ISequentialInStream *stream); + STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps); + + CHandler() { } +}; + +static const Byte kProps[] = +{ + kpidSize, + kpidPackSize +}; + +static const Byte kArcProps[] = +{ + kpidNumStreams, + kpidNumBlocks +}; + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/) +{ + return S_OK; +} + +STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems) +{ + *numItems = 1; + return S_OK; +} + +STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPackSize: if (_packSize_Defined) prop = _packSize; break; + case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break; + } + prop.Detach(value); + return S_OK; +} + +static const unsigned kSignatureCheckSize = 4; + +API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size) +{ + if (size < 4) + return k_IsArc_Res_NEED_MORE; + + // 0xFD2FB525 = 0.5 + // .. + // 0xFD2FB528 = 0.8 + UInt32 magic = GetUi32(p); + if (magic >= 0xFD2FB525 && magic <= 0xFD2FB528) + return k_IsArc_Res_YES; + + return k_IsArc_Res_NO; +} +} + +STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *) +{ + COM_TRY_BEGIN + Close(); + { + Byte buf[kSignatureCheckSize]; + RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize)); + if (IsArc_zstd(buf, kSignatureCheckSize) == k_IsArc_Res_NO) + return S_FALSE; + _isArc = true; + _stream = stream; + _seqStream = stream; + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + } + return S_OK; + COM_TRY_END +} + + +STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream) +{ + Close(); + _isArc = true; + _seqStream = stream; + return S_OK; +} + +STDMETHODIMP CHandler::Close() +{ + _isArc = false; + _dataAfterEnd = false; + _needMoreInput = false; + + _packSize_Defined = false; + _unpackSize_Defined = false; + + _packSize = 0; + + _seqStream.Release(); + _stream.Release(); + return S_OK; +} + +STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback) +{ + COM_TRY_BEGIN + if (numItems == 0) + return S_OK; + if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0)) + return E_INVALIDARG; + + if (_packSize_Defined) + extractCallback->SetTotal(_packSize); + + CMyComPtr realOutStream; + Int32 askMode = testMode ? + NExtract::NAskMode::kTest : + NExtract::NAskMode::kExtract; + RINOK(extractCallback->GetStream(0, &realOutStream, askMode)); + if (!testMode && !realOutStream) + return S_OK; + + extractCallback->PrepareOperation(askMode); + + Int32 opRes; + + { + + NCompress::NZSTD::CDecoder *decoderSpec = new NCompress::NZSTD::CDecoder; + CMyComPtr decoder = decoderSpec; + decoderSpec->SetInStream(_seqStream); + + CDummyOutStream *outStreamSpec = new CDummyOutStream; + CMyComPtr outStream(outStreamSpec); + outStreamSpec->SetStream(realOutStream); + outStreamSpec->Init(); + + realOutStream.Release(); + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(extractCallback, true); + + UInt64 packSize = 0; + UInt64 unpackedSize = 0; + + HRESULT result = S_OK; + + for (;;) + { + lps->InSize = packSize; + lps->OutSize = unpackedSize; + + RINOK(lps->SetCur()); + result = decoderSpec->CodeResume(outStream, &unpackedSize, progress); + UInt64 streamSize = decoderSpec->GetInputProcessedSize(); + + printf("streamsize=%d packsize=%d unpackedSize=%d\n", + streamSize, packSize, unpackedSize); + fflush(stdout); + + if (result != S_FALSE && result != S_OK) + return result; + + if (unpackedSize == 0) + break; + + if (streamSize == packSize) + { + // no new bytes in input stream, So it's good end of archive. + result = S_OK; + break; + } + + if (packSize > streamSize) + return E_FAIL; + + if (result != S_OK) + break; + } + + decoderSpec->ReleaseInStream(); + outStream.Release(); + + if (!_isArc) + opRes = NExtract::NOperationResult::kIsNotArc; + else if (_needMoreInput) + opRes = NExtract::NOperationResult::kUnexpectedEnd; + else if (_dataAfterEnd) + opRes = NExtract::NOperationResult::kDataAfterEnd; + else if (result == S_FALSE) + opRes = NExtract::NOperationResult::kDataError; + else if (result == S_OK) + opRes = NExtract::NOperationResult::kOK; + else + return result; + + } + + return extractCallback->SetOperationResult(opRes); + + COM_TRY_END +} + +static HRESULT UpdateArchive( + UInt64 unpackSize, + ISequentialOutStream *outStream, + const CProps &props, + IArchiveUpdateCallback *updateCallback) +{ + RINOK(updateCallback->SetTotal(unpackSize)); + CMyComPtr fileInStream; + RINOK(updateCallback->GetStream(0, &fileInStream)); + CLocalProgress *localProgressSpec = new CLocalProgress; + CMyComPtr localProgress = localProgressSpec; + localProgressSpec->Init(updateCallback, true); + NCompress::NZSTD::CEncoder *encoderSpec = new NCompress::NZSTD::CEncoder; + CMyComPtr encoder = encoderSpec; + RINOK(props.SetCoderProps(encoderSpec, NULL)); + RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress)); + return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK); +} + +STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + +STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *updateCallback) +{ + COM_TRY_BEGIN + + if (numItems != 1) + return E_INVALIDARG; + + Int32 newData, newProps; + UInt32 indexInArchive; + if (!updateCallback) + return E_FAIL; + RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive)); + + if ((newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop)); + if (prop.vt != VT_EMPTY) + if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE) + return E_INVALIDARG; + } + } + + if ((newData)) + { + UInt64 size; + { + NCOM::CPropVariant prop; + RINOK(updateCallback->GetProperty(0, kpidSize, &prop)); + if (prop.vt != VT_UI8) + return E_INVALIDARG; + size = prop.uhVal.QuadPart; + } + return UpdateArchive(size, outStream, _props, updateCallback); + } + + if (indexInArchive != 0) + return E_INVALIDARG; + + CLocalProgress *lps = new CLocalProgress; + CMyComPtr progress = lps; + lps->Init(updateCallback, true); + + CMyComPtr opCallback; + updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback); + if (opCallback) + { + RINOK(opCallback->ReportOperation( + NEventIndexType::kInArcIndex, 0, + NUpdateNotifyOp::kReplicate)) + } + + if (_stream) + RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL)); + + return NCompress::CopyStream(_stream, outStream, progress); + + COM_TRY_END +} + +STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) +{ + return _props.SetProperties(names, values, numProps); +} + +static const Byte k_Signature[] = "0xFD2FB525 .. 0xFD2FB528"; + +REGISTER_ARC_IO( + "zstd", "zst tzstd", "* .tar", 0x0e, + k_Signature, + 0, + NArcInfoFlags::kKeepName, + IsArc_zstd) + +}} diff --git a/CPP/7zip/Bundles/Format7zF/Arc.mak b/CPP/7zip/Bundles/Format7zF/Arc.mak index a7c6d645..75ab01a5 100644 --- a/CPP/7zip/Bundles/Format7zF/Arc.mak +++ b/CPP/7zip/Bundles/Format7zF/Arc.mak @@ -90,6 +90,7 @@ AR_OBJS = \ $O\XarHandler.obj \ $O\XzHandler.obj \ $O\ZHandler.obj \ + $O\ZstdHandler.obj \ AR_COMMON_OBJS = \ $O\CoderMixer2.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/resource.rc b/CPP/7zip/Bundles/Format7zF/resource.rc index 8e869662..c38d3542 100644 --- a/CPP/7zip/Bundles/Format7zF/resource.rc +++ b/CPP/7zip/Bundles/Format7zF/resource.rc @@ -28,10 +28,10 @@ MY_VERSION_INFO_DLL("7z Plugin", "7z") 22 ICON "../../Archive/Icons/ntfs.ico" 23 ICON "../../Archive/Icons/xz.ico" 24 ICON "../../Archive/Icons/squashfs.ico" - +25 ICON "../../Archive/Icons/zst.ico" STRINGTABLE BEGIN - 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24" + 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25" END diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp index 1d35c983..43d11cad 100644 --- a/CPP/7zip/Compress/ZstdDecoder.cpp +++ b/CPP/7zip/Compress/ZstdDecoder.cpp @@ -16,8 +16,7 @@ CDecoder::CDecoder(): _buffInSize(ZSTD_DStreamInSize()), _buffOutSize(ZSTD_DStreamOutSize()*4), _processedIn(0), - _processedOut(0), - _propsWereSet(false) + _processedOut(0) { _props.clear(); } @@ -66,7 +65,6 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) #endif memcpy(&_props, pProps, sizeof (DProps)); - _propsWereSet = true; return S_OK; } @@ -91,9 +89,6 @@ HRESULT CDecoder::CreateDecompressor() { size_t result; - if (!_propsWereSet) - return E_FAIL; - if (!_dstream) { _dstream = ZSTD_createDStream(); if (!_dstream) @@ -198,6 +193,13 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream /* finished */ if (input.pos == input.size) break; + + /* end of frame */ + if (result == 0) { + result = ZSTD_initDStream(_dstream); + if (ZSTD_isError(result)) + return ErrorOut(result); + } } } } @@ -266,6 +268,13 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize) /* finished */ if (input.pos == input.size) break; + + /* end of frame */ + if (result == 0) { + result = ZSTD_initDStream(_dstream); + if (ZSTD_isError(result)) + return ErrorOut(result); + } } } } diff --git a/CPP/7zip/Compress/ZstdDecoder.h b/CPP/7zip/Compress/ZstdDecoder.h index c31c08a5..ca63cb42 100644 --- a/CPP/7zip/Compress/ZstdDecoder.h +++ b/CPP/7zip/Compress/ZstdDecoder.h @@ -49,7 +49,6 @@ class CDecoder:public ICompressCoder, CMyComPtr < ISequentialInStream > _inStream; DProps _props; - bool _propsWereSet; ZSTD_DStream *_dstream; void *_buffIn; diff --git a/CPP/7zip/UI/Explorer/ContextMenu.cpp b/CPP/7zip/UI/Explorer/ContextMenu.cpp index ca058f4b..3326620b 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -804,7 +804,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, mi.fMask |= MIIM_CHECKMARKS; mi.wID = currentCommandID++; mi.hSubMenu = subMenu; - mi.StringValue.SetFromAscii("CRC SHA ZS"); + mi.StringValue.SetFromAscii("CRC SHA ZStandard"); mi.hbmpUnchecked = bitmap; CMenu menu;