xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-06 23:14:54 -06:00
Code Issues Packages Projects Releases Wiki Activity

feature release

Browse Source 
- you can open / crate .tar.zst files now
- direct compressing of files with zstd works also
- you can also open files compressed with pzstd
- next version will do also threaded compression like pzstd does
- zst files can be registred with 7-Zip ZS now
- update to latest zstd dev release v1.0.1
- the About Box will give you a hint, that this 7-Zip is not the default one
This commit is contained in:
Tino Reichardt
2016-09-15 17:57:28 +02:00
parent 55b9bd98b7
commit 00c0d31e31
18 changed files with 813 additions and 1611 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
C/7zVersion.h
View File

@@ -1,9 +1,9 @@
#define MY_VER_MAJOR 16 #define MY_VER_MAJOR 16
#define MY_VER_MINOR 02 #define MY_VER_MINOR 02
#define MY_VER_BUILD 0 #define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "16.02" #define MY_VERSION_NUMBERS "16.02 ZS"
#define MY_VERSION "16.02" #define MY_VERSION "16.02 ZS"
#define MY_DATE "2016-06-12" #define MY_DATE "2016-09-15"
#undef MY_COPYRIGHT #undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE #undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"

2
C/7zVersionTr.h
View File

@@ -3,7 +3,7 @@
#define MY_VER_BUILD 1 #define MY_VER_BUILD 1
#define MY_VERSION_NUMBERS "1.0.1" #define MY_VERSION_NUMBERS "1.0.1"
#define MY_VERSION "1.0.1" #define MY_VERSION "1.0.1"
#define MY_DATE "2016-09-04" #define MY_DATE "2016-09-15"
#undef MY_COPYRIGHT #undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE #undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Tino Reichardt" #define MY_AUTHOR_NAME "Tino Reichardt"

340
C/zstd/zbuff_compress.c
View File

@@ -1,340 +0,0 @@
/*
Buffered version of Zstd compression library
Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd homepage : http://www.zstd.net/
*/
/* *************************************
* Dependencies
***************************************/
#include <stdlib.h>
#include "error_private.h"
#include "zstd_internal.h" /* MIN, ZSTD_BLOCKHEADERSIZE, defaultCustomMem */
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
/* *************************************
* Constants
***************************************/
static size_t const ZBUFF_endFrameSize = ZSTD_BLOCKHEADERSIZE;
/*-***********************************************************
* Streaming compression
*
* A ZBUFF_CCtx object is required to track streaming operation.
* Use ZBUFF_createCCtx() and ZBUFF_freeCCtx() to create/release resources.
* Use ZBUFF_compressInit() to start a new compression operation.
* ZBUFF_CCtx objects can be reused multiple times.
*
* Use ZBUFF_compressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to call again the function with remaining input.
* The content of dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters or change dst .
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to improve latency)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressFlush() can be used to instruct ZBUFF to compress and output whatever remains within its buffer.
* Note that it will not output more than *dstCapacityPtr.
* Therefore, some content might still be left into its internal buffer if dst buffer is too small.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* ZBUFF_compressEnd() instructs to finish a frame.
* It will perform a flush and write frame epilogue.
* Similar to ZBUFF_compressFlush(), it may not be able to output the entire internal buffer content if *dstCapacityPtr is too small.
* @return : nb of bytes still present into internal buffer (0 if it's empty)
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory)
* input : ZSTD_BLOCKSIZE_MAX (128 KB), internal unit size, it improves latency to use this value.
* output : ZSTD_compressBound(ZSTD_BLOCKSIZE_MAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize : ensures it's always possible to write/flush/end a full block at best speed.
* ***********************************************************/
typedef enum { ZBUFFcs_init, ZBUFFcs_load, ZBUFFcs_flush, ZBUFFcs_final } ZBUFF_cStage;
/* *** Resources *** */
struct ZBUFF_CCtx_s {
ZSTD_CCtx* zc;
char* inBuff;
size_t inBuffSize;
size_t inToCompress;
size_t inBuffPos;
size_t inBuffTarget;
size_t blockSize;
char* outBuff;
size_t outBuffSize;
size_t outBuffContentSize;
size_t outBuffFlushedSize;
ZBUFF_cStage stage;
U32 checksum;
U32 frameEnded;
ZSTD_customMem customMem;
}; /* typedef'd tp ZBUFF_CCtx within "zstd_buffered.h" */
ZBUFF_CCtx* ZBUFF_createCCtx(void)
{
return ZBUFF_createCCtx_advanced(defaultCustomMem);
}
ZBUFF_CCtx* ZBUFF_createCCtx_advanced(ZSTD_customMem customMem)
{
ZBUFF_CCtx* zbc;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zbc = (ZBUFF_CCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_CCtx));
if (zbc==NULL) return NULL;
memset(zbc, 0, sizeof(ZBUFF_CCtx));
memcpy(&zbc->customMem, &customMem, sizeof(ZSTD_customMem));
zbc->zc = ZSTD_createCCtx_advanced(customMem);
if (zbc->zc == NULL) { ZBUFF_freeCCtx(zbc); return NULL; }
return zbc;
}
size_t ZBUFF_freeCCtx(ZBUFF_CCtx* zbc)
{
if (zbc==NULL) return 0; /* support free on NULL */
ZSTD_freeCCtx(zbc->zc);
if (zbc->inBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff);
if (zbc->outBuff) zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff);
zbc->customMem.customFree(zbc->customMem.opaque, zbc);
return 0;
}
/* ====== Initialization ====== */
size_t ZBUFF_compressInit_advanced(ZBUFF_CCtx* zbc,
const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize)
{
/* allocate buffers */
{ size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog;
if (zbc->inBuffSize < neededInBuffSize) {
zbc->inBuffSize = neededInBuffSize;
zbc->customMem.customFree(zbc->customMem.opaque, zbc->inBuff); /* should not be necessary */
zbc->inBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, neededInBuffSize);
if (zbc->inBuff == NULL) return ERROR(memory_allocation);
}
zbc->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize);
}
if (zbc->outBuffSize < ZSTD_compressBound(zbc->blockSize)+1) {
zbc->outBuffSize = ZSTD_compressBound(zbc->blockSize)+1;
zbc->customMem.customFree(zbc->customMem.opaque, zbc->outBuff); /* should not be necessary */
zbc->outBuff = (char*)zbc->customMem.customAlloc(zbc->customMem.opaque, zbc->outBuffSize);
if (zbc->outBuff == NULL) return ERROR(memory_allocation);
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(zbc->zc, dict, dictSize, params, pledgedSrcSize);
if (ZSTD_isError(errorCode)) return errorCode; }
zbc->inToCompress = 0;
zbc->inBuffPos = 0;
zbc->inBuffTarget = zbc->blockSize;
zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_load;
zbc->checksum = params.fParams.checksumFlag > 0;
zbc->frameEnded = 0;
return 0; /* ready to go */
}
size_t ZBUFF_compressInitDictionary(ZBUFF_CCtx* zbc, const void* dict, size_t dictSize, int compressionLevel)
{
ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize);
return ZBUFF_compressInit_advanced(zbc, dict, dictSize, params, 0);
}
size_t ZBUFF_compressInit(ZBUFF_CCtx* zbc, int compressionLevel)
{
return ZBUFF_compressInitDictionary(zbc, NULL, 0, compressionLevel);
}
/* internal util function */
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
/* ====== Compression ====== */
typedef enum { zbf_gather, zbf_flush, zbf_end } ZBUFF_flush_e;
static size_t ZBUFF_compressContinue_generic(ZBUFF_CCtx* zbc,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr,
ZBUFF_flush_e const flush)
{
U32 someMoreWork = 1;
const char* const istart = (const char*)src;
const char* const iend = istart + *srcSizePtr;
const char* ip = istart;
char* const ostart = (char*)dst;
char* const oend = ostart + *dstCapacityPtr;
char* op = ostart;
while (someMoreWork) {
switch(zbc->stage)
{
case ZBUFFcs_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */
case ZBUFFcs_load:
/* complete inBuffer */
{ size_t const toLoad = zbc->inBuffTarget - zbc->inBuffPos;
size_t const loaded = ZBUFF_limitCopy(zbc->inBuff + zbc->inBuffPos, toLoad, ip, iend-ip);
zbc->inBuffPos += loaded;
ip += loaded;
if ( (zbc->inBuffPos==zbc->inToCompress) || (!flush && (toLoad != loaded)) ) {
someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */
} }
/* compress current block (note : this stage cannot be stopped in the middle) */
{ void* cDst;
size_t cSize;
size_t const iSize = zbc->inBuffPos - zbc->inToCompress;
size_t oSize = oend-op;
if (oSize >= ZSTD_compressBound(iSize))
cDst = op; /* compress directly into output buffer (avoid flush stage) */
else
cDst = zbc->outBuff, oSize = zbc->outBuffSize;
cSize = (flush == zbf_end) ?
ZSTD_compressEnd(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize) :
ZSTD_compressContinue(zbc->zc, cDst, oSize, zbc->inBuff + zbc->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize;
if (flush == zbf_end) zbc->frameEnded = 1;
/* prepare next block */
zbc->inBuffTarget = zbc->inBuffPos + zbc->blockSize;
if (zbc->inBuffTarget > zbc->inBuffSize)
zbc->inBuffPos = 0, zbc->inBuffTarget = zbc->blockSize; /* note : inBuffSize >= blockSize */
zbc->inToCompress = zbc->inBuffPos;
if (cDst == op) { op += cSize; break; } /* no need to flush */
zbc->outBuffContentSize = cSize;
zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_flush; /* continue to flush stage */
}
case ZBUFFcs_flush:
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
op += flushed;
zbc->outBuffFlushedSize += flushed;
if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */
zbc->outBuffContentSize = zbc->outBuffFlushedSize = 0;
zbc->stage = ZBUFFcs_load;
break;
}
case ZBUFFcs_final:
someMoreWork = 0; /* do nothing */
break;
default:
return ERROR(GENERIC); /* impossible */
}
}
*srcSizePtr = ip - istart;
*dstCapacityPtr = op - ostart;
if (zbc->frameEnded) return 0;
{ size_t hintInSize = zbc->inBuffTarget - zbc->inBuffPos;
if (hintInSize==0) hintInSize = zbc->blockSize;
return hintInSize;
}
}
size_t ZBUFF_compressContinue(ZBUFF_CCtx* zbc,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr)
{
return ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, src, srcSizePtr, zbf_gather);
}
/* ====== Finalize ====== */
size_t ZBUFF_compressFlush(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
{
size_t srcSize = 0;
ZBUFF_compressContinue_generic(zbc, dst, dstCapacityPtr, &srcSize, &srcSize, zbf_flush); /* use a valid src address instead of NULL */
return zbc->outBuffContentSize - zbc->outBuffFlushedSize;
}
size_t ZBUFF_compressEnd(ZBUFF_CCtx* zbc, void* dst, size_t* dstCapacityPtr)
{
BYTE* const ostart = (BYTE*)dst;
BYTE* const oend = ostart + *dstCapacityPtr;
BYTE* op = ostart;
if (zbc->stage != ZBUFFcs_final) {
/* flush whatever remains */
size_t outSize = *dstCapacityPtr;
size_t srcSize = 0;
size_t const notEnded = ZBUFF_compressContinue_generic(zbc, dst, &outSize, &srcSize, &srcSize, zbf_end); /* use a valid address instead of NULL */
size_t const remainingToFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
op += outSize;
if (remainingToFlush) {
*dstCapacityPtr = op-ostart;
return remainingToFlush + ZBUFF_endFrameSize + (zbc->checksum * 4);
}
/* create epilogue */
zbc->stage = ZBUFFcs_final;
zbc->outBuffContentSize = !notEnded ? 0 :
ZSTD_compressEnd(zbc->zc, zbc->outBuff, zbc->outBuffSize, NULL, 0); /* write epilogue into outBuff */
}
/* flush epilogue */
{ size_t const toFlush = zbc->outBuffContentSize - zbc->outBuffFlushedSize;
size_t const flushed = ZBUFF_limitCopy(op, oend-op, zbc->outBuff + zbc->outBuffFlushedSize, toFlush);
op += flushed;
zbc->outBuffFlushedSize += flushed;
*dstCapacityPtr = op-ostart;
if (toFlush==flushed) zbc->stage = ZBUFFcs_init; /* end reached */
return toFlush - flushed;
}
}
/* *************************************
* Tool functions
***************************************/
size_t ZBUFF_recommendedCInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZBUFF_recommendedCOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + ZBUFF_endFrameSize; }

297
C/zstd/zbuff_decompress.c
View File

@@ -1,297 +0,0 @@
/*
Buffered version of Zstd compression library
Copyright (C) 2015-2016, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- zstd homepage : http://www.zstd.net/
*/
/* *************************************
* Dependencies
***************************************/
#include <stdlib.h>
#include "error_private.h"
#include "zstd_internal.h" /* MIN, ZSTD_blockHeaderSize, ZSTD_BLOCKSIZE_MAX */
#define ZBUFF_STATIC_LINKING_ONLY
#include "zbuff.h"
/*-***************************************************************************
* Streaming decompression howto
*
* A ZBUFF_DCtx object is required to track streaming operations.
* Use ZBUFF_createDCtx() and ZBUFF_freeDCtx() to create/release resources.
* Use ZBUFF_decompressInit() to start a new decompression operation,
* or ZBUFF_decompressInitDictionary() if decompression requires a dictionary.
* Note that ZBUFF_DCtx objects can be re-init multiple times.
*
* Use ZBUFF_decompressContinue() repetitively to consume your input.
* *srcSizePtr and *dstCapacityPtr can be any size.
* The function will report how many bytes were read or written by modifying *srcSizePtr and *dstCapacityPtr.
* Note that it may not consume the entire input, in which case it's up to the caller to present remaining input again.
* The content of @dst will be overwritten (up to *dstCapacityPtr) at each function call, so save its content if it matters, or change @dst.
* @return : a hint to preferred nb of bytes to use as input for next function call (it's only a hint, to help latency),
* or 0 when a frame is completely decoded,
* or an error code, which can be tested using ZBUFF_isError().
*
* Hint : recommended buffer sizes (not compulsory) : ZBUFF_recommendedDInSize() and ZBUFF_recommendedDOutSize()
* output : ZBUFF_recommendedDOutSize==128 KB block size is the internal unit, it ensures it's always possible to write a full block when decoded.
* input : ZBUFF_recommendedDInSize == 128KB + 3;
* just follow indications from ZBUFF_decompressContinue() to minimize latency. It should always be <= 128 KB + 3 .
* *******************************************************************************/
typedef enum { ZBUFFds_init, ZBUFFds_loadHeader,
ZBUFFds_read, ZBUFFds_load, ZBUFFds_flush } ZBUFF_dStage;
/* *** Resource management *** */
struct ZBUFF_DCtx_s {
ZSTD_DCtx* zd;
ZSTD_frameParams fParams;
ZBUFF_dStage stage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
size_t lhSize;
ZSTD_customMem customMem;
}; /* typedef'd to ZBUFF_DCtx within "zstd_buffered.h" */
ZBUFF_DCtx* ZBUFF_createDCtx(void)
{
return ZBUFF_createDCtx_advanced(defaultCustomMem);
}
ZBUFF_DCtx* ZBUFF_createDCtx_advanced(ZSTD_customMem customMem)
{
ZBUFF_DCtx* zbd;
if (!customMem.customAlloc && !customMem.customFree)
customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree)
return NULL;
zbd = (ZBUFF_DCtx*)customMem.customAlloc(customMem.opaque, sizeof(ZBUFF_DCtx));
if (zbd==NULL) return NULL;
memset(zbd, 0, sizeof(ZBUFF_DCtx));
memcpy(&zbd->customMem, &customMem, sizeof(ZSTD_customMem));
zbd->zd = ZSTD_createDCtx_advanced(customMem);
if (zbd->zd == NULL) { ZBUFF_freeDCtx(zbd); return NULL; }
zbd->stage = ZBUFFds_init;
return zbd;
}
size_t ZBUFF_freeDCtx(ZBUFF_DCtx* zbd)
{
if (zbd==NULL) return 0; /* support free on null */
ZSTD_freeDCtx(zbd->zd);
if (zbd->inBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
if (zbd->outBuff) zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
zbd->customMem.customFree(zbd->customMem.opaque, zbd);
return 0;
}
/* *** Initialization *** */
size_t ZBUFF_decompressInitDictionary(ZBUFF_DCtx* zbd, const void* dict, size_t dictSize)
{
zbd->stage = ZBUFFds_loadHeader;
zbd->lhSize = zbd->inPos = zbd->outStart = zbd->outEnd = 0;
return ZSTD_decompressBegin_usingDict(zbd->zd, dict, dictSize);
}
size_t ZBUFF_decompressInit(ZBUFF_DCtx* zbd)
{
return ZBUFF_decompressInitDictionary(zbd, NULL, 0);
}
/* internal util function */
MEM_STATIC size_t ZBUFF_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{
size_t const length = MIN(dstCapacity, srcSize);
memcpy(dst, src, length);
return length;
}
/* *** Decompression *** */
size_t ZBUFF_decompressContinue(ZBUFF_DCtx* zbd,
void* dst, size_t* dstCapacityPtr,
const void* src, size_t* srcSizePtr)
{
const char* const istart = (const char*)src;
const char* const iend = istart + *srcSizePtr;
const char* ip = istart;
char* const ostart = (char*)dst;
char* const oend = ostart + *dstCapacityPtr;
char* op = ostart;
U32 someMoreWork = 1;
while (someMoreWork) {
switch(zbd->stage)
{
case ZBUFFds_init :
return ERROR(init_missing);
case ZBUFFds_loadHeader :
{ size_t const hSize = ZSTD_getFrameParams(&(zbd->fParams), zbd->headerBuffer, zbd->lhSize);
if (ZSTD_isError(hSize)) return hSize;
if (hSize != 0) { /* need more input */
size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */
if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */
memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip);
zbd->lhSize += iend-ip;
*dstCapacityPtr = 0;
return (hSize - zbd->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
}
memcpy(zbd->headerBuffer + zbd->lhSize, ip, toLoad); zbd->lhSize = hSize; ip += toLoad;
break;
} }
/* Consume header */
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zbd->zd); /* == ZSTD_frameHeaderSize_min */
size_t const h1Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer, h1Size);
if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */
if (h1Size < zbd->lhSize) { /* long header */
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zbd->zd);
size_t const h2Result = ZSTD_decompressContinue(zbd->zd, NULL, 0, zbd->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} }
zbd->fParams.windowSize = MAX(zbd->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
/* Frame header instruct buffer sizes */
{ size_t const blockSize = MIN(zbd->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zbd->fParams.windowSize + blockSize;
zbd->blockSize = blockSize;
if (zbd->inBuffSize < blockSize) {
zbd->customMem.customFree(zbd->customMem.opaque, zbd->inBuff);
zbd->inBuffSize = blockSize;
zbd->inBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, blockSize);
if (zbd->inBuff == NULL) return ERROR(memory_allocation);
}
if (zbd->outBuffSize < neededOutSize) {
zbd->customMem.customFree(zbd->customMem.opaque, zbd->outBuff);
zbd->outBuffSize = neededOutSize;
zbd->outBuff = (char*)zbd->customMem.customAlloc(zbd->customMem.opaque, neededOutSize);
if (zbd->outBuff == NULL) return ERROR(memory_allocation);
} }
zbd->stage = ZBUFFds_read;
/* pass-through */
case ZBUFFds_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
if (neededInSize==0) { /* end of frame */
zbd->stage = ZBUFFds_init;
someMoreWork = 0;
break;
}
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, (isSkipFrame ? 0 : zbd->outBuffSize - zbd->outStart),
ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
break;
}
if (ip==iend) { someMoreWork = 0; break; } /* no more input */
zbd->stage = ZBUFFds_load;
/* pass-through */
}
case ZBUFFds_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zbd->zd);
size_t const toLoad = neededInSize - zbd->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize;
if (toLoad > zbd->inBuffSize - zbd->inPos) return ERROR(corruption_detected); /* should never happen */
loadedSize = ZBUFF_limitCopy(zbd->inBuff + zbd->inPos, toLoad, ip, iend-ip);
ip += loadedSize;
zbd->inPos += loadedSize;
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zbd->zd);
size_t const decodedSize = ZSTD_decompressContinue(zbd->zd,
zbd->outBuff + zbd->outStart, zbd->outBuffSize - zbd->outStart,
zbd->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize;
zbd->inPos = 0; /* input is consumed */
if (!decodedSize && !isSkipFrame) { zbd->stage = ZBUFFds_read; break; } /* this was just a header */
zbd->outEnd = zbd->outStart + decodedSize;
zbd->stage = ZBUFFds_flush;
/* pass-through */
} }
case ZBUFFds_flush:
{ size_t const toFlushSize = zbd->outEnd - zbd->outStart;
size_t const flushedSize = ZBUFF_limitCopy(op, oend-op, zbd->outBuff + zbd->outStart, toFlushSize);
op += flushedSize;
zbd->outStart += flushedSize;
if (flushedSize == toFlushSize) { /* flush completed */
zbd->stage = ZBUFFds_read;
if (zbd->outStart + zbd->blockSize > zbd->outBuffSize)
zbd->outStart = zbd->outEnd = 0;
break;
}
/* cannot flush everything */
someMoreWork = 0;
break;
}
default: return ERROR(GENERIC); /* impossible */
} }
/* result */
*srcSizePtr = ip-istart;
*dstCapacityPtr = op-ostart;
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zbd->zd);
if (!nextSrcSizeHint) return (zbd->outEnd != zbd->outStart); /* return 0 only if fully flushed too */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zbd->zd) == ZSTDnit_block);
if (zbd->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zbd->inPos; /* already loaded*/
return nextSrcSizeHint;
}
}
/* *************************************
* Tool functions
***************************************/
size_t ZBUFF_recommendedDInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize /* block header size*/ ; }
size_t ZBUFF_recommendedDOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }

93
C/zstd/zstd.h
View File

@@ -48,40 +48,41 @@ ZSTDLIB_API unsigned ZSTD_versionNumber (void);
* Simple API * Simple API
***************************************/ ***************************************/
/*! ZSTD_compress() : /*! ZSTD_compress() :
Compresses `src` buffer into already allocated `dst`. Compresses `src` content as a single zstd compressed frame into already allocated `dst`.
Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`.
@return : the number of bytes written into `dst` (<= `dstCapacity), @return : compressed size written into `dst` (<= `dstCapacity),
or an error code if it fails (which can be tested using ZSTD_isError()) */ or an error code if it fails (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
int compressionLevel); int compressionLevel);
/*! ZSTD_getDecompressedSize() :
* @return : decompressed size as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 2 : decompressed size is an optional field, that may not be present.
* When `return==0`, data to decompress can have any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application may rely on its own implied limits.
* (For example, application data could be necessarily cut into blocks <= 16 KB).
* note 3 : decompressed size could be wrong or intentionally modified !
* Always ensure result fits within application's authorized limits !
* Each application can set its own limits.
* note 4 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_decompress() : /*! ZSTD_decompress() :
`compressedSize` : must be the _exact_ size of compressed input, otherwise decompression will fail. `compressedSize` : must be the _exact_ size of a single compressed frame.
`dstCapacity` must be equal or larger than originalSize (see ZSTD_getDecompressedSize() ). `dstCapacity` is an upper bound of originalSize.
If originalSize is unknown, and if there is no implied application-specific limitations, If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
it's preferable to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`), @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
or an errorCode if it fails (which can be tested using ZSTD_isError()) */ or an errorCode if it fails (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize); const void* src, size_t compressedSize);
/*! ZSTD_getDecompressedSize() :
* 'src' is the start of a zstd compressed frame.
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
* When `return==0`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
* (For example, data may be necessarily cut into blocks <= 16 KB).
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
* note 3 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure result fits within application's authorized limits.
* Each application can set its own limits.
* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */
ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize);
/*====== Helper functions ======*/ /*====== Helper functions ======*/
ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */
@@ -200,7 +201,7 @@ typedef struct ZSTD_outBuffer_s {
* Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary. * Use ZSTD_initCStream_usingDict() for a compression which requires a dictionary.
* *
* Use ZSTD_compressStream() repetitively to consume input stream. * Use ZSTD_compressStream() repetitively to consume input stream.
* The function will automatically update both `pos`. * The function will automatically update both `pos` fields.
* Note that it may not consume the entire input, in which case `pos < size`, * Note that it may not consume the entire input, in which case `pos < size`,
* and it's up to the caller to present again remaining data. * and it's up to the caller to present again remaining data.
* @return : a size hint, preferred nb of bytes to use as input for next function call * @return : a size hint, preferred nb of bytes to use as input for next function call
@@ -229,7 +230,7 @@ ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer */ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
@@ -248,16 +249,17 @@ ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
* *
* Use ZSTD_initDStream() to start a new decompression operation, * Use ZSTD_initDStream() to start a new decompression operation,
* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. * or ZSTD_initDStream_usingDict() if decompression requires a dictionary.
* @return : recommended first input size
* *
* Use ZSTD_decompressStream() repetitively to consume your input. * Use ZSTD_decompressStream() repetitively to consume your input.
* The function will update both `pos`. * The function will update both `pos` fields.
* Note that it may not consume the entire input (pos < size), * If `input.pos < input.size`, some input has not been consumed.
* in which case it's up to the caller to present remaining input again. * It's up to the caller to present again remaining data.
* If `output.pos < output.size`, decoder has flushed everything it could.
* @return : 0 when a frame is completely decoded and fully flushed, * @return : 0 when a frame is completely decoded and fully flushed,
* 1 when there is still some data left within internal buffer to flush, * an error code, which can be tested using ZSTD_isError(),
* >1 when more data is expected, with value being a suggested next input size (it's just a hint, which helps latency, any size is accepted), * any other value > 0, which means there is still some work to do to complete the frame.
* or an error code, which can be tested using ZSTD_isError(). * The return value is a suggested next input size (just an hint, to help latency).
*
* *******************************************************************************/ * *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream; typedef struct ZSTD_DStream_s ZSTD_DStream;
@@ -265,7 +267,7 @@ ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */
ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer */ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
@@ -296,13 +298,14 @@ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* outp
#define ZSTD_HASHLOG3_MAX 17 #define ZSTD_HASHLOG3_MAX 17
#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) #define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1 #define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7 #define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */
#define ZSTD_SEARCHLENGTH_MIN 3 #define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */
#define ZSTD_TARGETLENGTH_MIN 4 #define ZSTD_TARGETLENGTH_MIN 4
#define ZSTD_TARGETLENGTH_MAX 999 #define ZSTD_TARGETLENGTH_MAX 999
#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ #define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */
static const size_t ZSTD_frameHeaderSize_min = 5; static const size_t ZSTD_frameHeaderSize_prefix = 5;
static const size_t ZSTD_frameHeaderSize_min = 6;
static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX;
static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */
@@ -349,14 +352,18 @@ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams);
* Create a ZSTD compression context using external alloc and free functions */ * Create a ZSTD compression context using external alloc and free functions */
ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_sizeofCCtx() :
* Gives the amount of memory used by a given ZSTD_CCtx */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
/*! ZSTD_createCDict_advanced() : /*! ZSTD_createCDict_advanced() :
* Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */
ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize,
ZSTD_parameters params, ZSTD_customMem customMem); ZSTD_parameters params, ZSTD_customMem customMem);
/*! ZSTD_sizeofCCtx() : /*! ZSTD_sizeof_CDict() :
* Gives the amount of memory used by a given ZSTD_CCtx */ * Gives the amount of memory used by a given ZSTD_sizeof_CDict */
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict);
/*! ZSTD_getParams() : /*! ZSTD_getParams() :
* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`. * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of a `ZSTD_compressionParameters`.
@@ -396,10 +403,14 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void);
* Create a ZSTD decompression context using external alloc and free functions */ * Create a ZSTD decompression context using external alloc and free functions */
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem);
/*! ZSTD_sizeofDCtx() : /*! ZSTD_sizeof_DCtx() :
* Gives the amount of memory used by a given ZSTD_DCtx */ * Gives the amount of memory used by a given ZSTD_DCtx */
ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
/*! ZSTD_sizeof_DDict() :
* Gives the amount of memory used by a given ZSTD_DDict */
ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict);
/* ****************************************************************** /* ******************************************************************
* Advanced Streaming functions * Advanced Streaming functions
@@ -410,7 +421,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
@@ -421,6 +433,7 @@ typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e;
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize);
ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue);
ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */
ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);

324
C/zstd/zstd_compress.c
View File

@@ -8,33 +8,13 @@
*/ */
/*-*******************************************************
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
#include <string.h> /* memset */ #include <string.h> /* memset */
#include "mem.h" #include "mem.h"
#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
#include "xxhash.h" /* XXH_reset, update, digest */ #include "xxhash.h" /* XXH_reset, update, digest */
#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */
#include "fse.h" #include "fse.h"
#define HUF_STATIC_LINKING_ONLY #define HUF_STATIC_LINKING_ONLY
@@ -119,7 +99,7 @@ ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem)
cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem);
if (!cctx) return NULL; if (!cctx) return NULL;
memset(cctx, 0, sizeof(ZSTD_CCtx)); memset(cctx, 0, sizeof(ZSTD_CCtx));
memcpy(&(cctx->customMem), &customMem, sizeof(ZSTD_customMem)); memcpy(&(cctx->customMem), &customMem, sizeof(customMem));
return cctx; return cctx;
} }
@@ -133,6 +113,7 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx)
size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx)
{ {
if (cctx==NULL) return 0; /* support sizeof on NULL */
return sizeof(*cctx) + cctx->workSpaceSize; return sizeof(*cctx) + cctx->workSpaceSize;
} }
@@ -142,19 +123,17 @@ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface *
} }
#define CLAMP(val,min,max) { if (val<min) val=min; else if (val>max) val=max; }
#define CLAMPCHECK(val,min,max) { if ((val<min) || (val>max)) return ERROR(compressionParameter_unsupported); }
/** ZSTD_checkParams() : /** ZSTD_checkParams() :
ensure param values remain within authorized range. ensure param values remain within authorized range.
@return : 0, or an error code if one value is beyond authorized range */ @return : 0, or an error code if one value is beyond authorized range */
size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams)
{ {
# define CLAMPCHECK(val,min,max) { if ((val<min) | (val>max)) return ERROR(compressionParameter_unsupported); }
CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX);
CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX);
CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX);
CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX);
{ U32 const searchLengthMin = (cParams.strategy == ZSTD_fast || cParams.strategy == ZSTD_greedy) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; { U32 const searchLengthMin = ((cParams.strategy == ZSTD_fast) | (cParams.strategy == ZSTD_greedy)) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN;
U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1;
CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); }
CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX);
@@ -171,7 +150,7 @@ size_t ZSTD_checkCParams_advanced(ZSTD_compressionParameters cParams, U64 srcSiz
if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported); if (cParams.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) return ERROR(compressionParameter_unsupported);
if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */ if (srcSize <= (1ULL << cParams.windowLog)) cParams.windowLog = ZSTD_WINDOWLOG_MIN; /* fake value - temporary work around */
if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */ if (srcSize <= (1ULL << cParams.chainLog)) cParams.chainLog = ZSTD_CHAINLOG_MIN; /* fake value - temporary work around */
if ((srcSize <= (1ULL << cParams.hashLog)) && ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */ if ((srcSize <= (1ULL << cParams.hashLog)) & ((U32)cParams.strategy < (U32)ZSTD_btlazy2)) cParams.hashLog = ZSTD_HASHLOG_MIN; /* fake value - temporary work around */
return ZSTD_checkCParams(cParams); return ZSTD_checkCParams(cParams);
} }
@@ -190,16 +169,16 @@ ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, u
{ U32 const minSrcSize = (srcSize==0) ? 500 : 0; { U32 const minSrcSize = (srcSize==0) ? 500 : 0;
U64 const rSize = srcSize + dictSize + minSrcSize; U64 const rSize = srcSize + dictSize + minSrcSize;
if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) { if (rSize < ((U64)1<<ZSTD_WINDOWLOG_MAX)) {
U32 const srcLog = ZSTD_highbit32((U32)(rSize)-1) + 1; U32 const srcLog = MAX(4, ZSTD_highbit32((U32)(rSize)-1) + 1);
if (cPar.windowLog > srcLog) cPar.windowLog = srcLog; if (cPar.windowLog > srcLog) cPar.windowLog = srcLog;
} } } }
if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog;
{ U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) || (cPar.strategy == ZSTD_btopt); { U32 const btPlus = (cPar.strategy == ZSTD_btlazy2) | (cPar.strategy == ZSTD_btopt);
U32 const maxChainLog = cPar.windowLog+btPlus; U32 const maxChainLog = cPar.windowLog+btPlus;
if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */ if (cPar.chainLog > maxChainLog) cPar.chainLog = maxChainLog; } /* <= ZSTD_CHAINLOG_MAX */
if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */
if ((cPar.hashLog < ZSTD_HASHLOG_MIN) && ( (U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */ if ((cPar.hashLog < ZSTD_HASHLOG_MIN) & ((U32)cPar.strategy >= (U32)ZSTD_btlazy2)) cPar.hashLog = ZSTD_HASHLOG_MIN; /* required to ensure collision resistance in bt */
return cPar; return cPar;
} }
@@ -226,81 +205,116 @@ size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams)
return sizeof(ZSTD_CCtx) + neededSpace; return sizeof(ZSTD_CCtx) + neededSpace;
} }
static U32 ZSTD_equivalentParams(ZSTD_parameters param1, ZSTD_parameters param2)
{
return (param1.cParams.hashLog == param2.cParams.hashLog)
& (param1.cParams.chainLog == param2.cParams.chainLog)
& (param1.cParams.strategy == param2.cParams.strategy)
& ((param1.cParams.searchLength==3) == (param2.cParams.searchLength==3));
}
/*! ZSTD_continueCCtx() :
reuse CCtx without reset (note : requires no dictionary) */
static size_t ZSTD_continueCCtx(ZSTD_CCtx* cctx, ZSTD_parameters params, U64 frameContentSize)
{
U32 const end = (U32)(cctx->nextSrc - cctx->base);
cctx->params = params;
cctx->frameContentSize = frameContentSize;
cctx->lowLimit = end;
cctx->dictLimit = end;
cctx->nextToUpdate = end+1;
cctx->stage = ZSTDcs_init;
cctx->dictID = 0;
cctx->loadedDictEnd = 0;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) cctx->rep[i] = repStartValue[i]; }
cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */
XXH64_reset(&cctx->xxhState, 0);
return 0;
}
typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e;
/*! ZSTD_resetCCtx_advanced() : /*! ZSTD_resetCCtx_advanced() :
note : 'params' is expected to be validated */ note : 'params' must be validated */
static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc,
ZSTD_parameters params, U64 frameContentSize, ZSTD_parameters params, U64 frameContentSize,
U32 reset) ZSTD_compResetPolicy_e const crp)
{ /* note : params considered validated here */ {
size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); if (crp == ZSTDcrp_continue)
U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; if (ZSTD_equivalentParams(params, zc->params))
size_t const maxNbSeq = blockSize / divider; return ZSTD_continueCCtx(zc, params, frameContentSize);
size_t const tokenSpace = blockSize + 11*maxNbSeq;
size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
size_t const hSize = ((size_t)1) << params.cParams.hashLog;
U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
size_t const h3Size = ((size_t)1) << hashLog3;
size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
void* ptr;
/* Check if workSpace is large enough, alloc a new one if needed */ { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog);
{ size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32) U32 const divider = (params.cParams.searchLength==3) ? 3 : 4;
+ (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t)); size_t const maxNbSeq = blockSize / divider;
size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace size_t const tokenSpace = blockSize + 11*maxNbSeq;
+ ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0); size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog);
if (zc->workSpaceSize < neededSpace) { size_t const hSize = ((size_t)1) << params.cParams.hashLog;
ZSTD_free(zc->workSpace, zc->customMem); U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog);
zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); size_t const h3Size = ((size_t)1) << hashLog3;
if (zc->workSpace == NULL) return ERROR(memory_allocation); size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32);
zc->workSpaceSize = neededSpace; void* ptr;
} }
if (reset) memset(zc->workSpace, 0, tableSpace ); /* reset only tables */ /* Check if workSpace is large enough, alloc a new one if needed */
XXH64_reset(&zc->xxhState, 0); { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<<Litbits))*sizeof(U32)
zc->hashLog3 = hashLog3; + (ZSTD_OPT_NUM+1)*(sizeof(ZSTD_match_t) + sizeof(ZSTD_optimal_t));
zc->hashTable = (U32*)(zc->workSpace); size_t const neededSpace = tableSpace + (256*sizeof(U32)) /* huffTable */ + tokenSpace
zc->chainTable = zc->hashTable + hSize; + ((params.cParams.strategy == ZSTD_btopt) ? optSpace : 0);
zc->hashTable3 = zc->chainTable + chainSize; if (zc->workSpaceSize < neededSpace) {
ptr = zc->hashTable3 + h3Size; ZSTD_free(zc->workSpace, zc->customMem);
zc->hufTable = (HUF_CElt*)ptr; zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem);
zc->flagStaticTables = 0; if (zc->workSpace == NULL) return ERROR(memory_allocation);
ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ zc->workSpaceSize = neededSpace;
} }
zc->nextToUpdate = 1; if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */
zc->nextSrc = NULL; XXH64_reset(&zc->xxhState, 0);
zc->base = NULL; zc->hashLog3 = hashLog3;
zc->dictBase = NULL; zc->hashTable = (U32*)(zc->workSpace);
zc->dictLimit = 0; zc->chainTable = zc->hashTable + hSize;
zc->lowLimit = 0; zc->hashTable3 = zc->chainTable + chainSize;
zc->params = params; ptr = zc->hashTable3 + h3Size;
zc->blockSize = blockSize; zc->hufTable = (HUF_CElt*)ptr;
zc->frameContentSize = frameContentSize; zc->flagStaticTables = 0;
{ int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; } ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */
if (params.cParams.strategy == ZSTD_btopt) { zc->nextToUpdate = 1;
zc->seqStore.litFreq = (U32*)ptr; zc->nextSrc = NULL;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits); zc->base = NULL;
zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); zc->dictBase = NULL;
zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); zc->dictLimit = 0;
ptr = zc->seqStore.offCodeFreq + (MaxOff+1); zc->lowLimit = 0;
zc->seqStore.matchTable = (ZSTD_match_t*)ptr; zc->params = params;
ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; zc->blockSize = blockSize;
zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; zc->frameContentSize = frameContentSize;
ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; { int i; for (i=0; i<ZSTD_REP_NUM; i++) zc->rep[i] = repStartValue[i]; }
zc->seqStore.litLengthSum = 0;
if (params.cParams.strategy == ZSTD_btopt) {
zc->seqStore.litFreq = (U32*)ptr;
zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<<Litbits);
zc->seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1);
zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1);
ptr = zc->seqStore.offCodeFreq + (MaxOff+1);
zc->seqStore.matchTable = (ZSTD_match_t*)ptr;
ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1;
zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr;
ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1;
zc->seqStore.litLengthSum = 0;
}
zc->seqStore.sequencesStart = (seqDef*)ptr;
ptr = zc->seqStore.sequencesStart + maxNbSeq;
zc->seqStore.llCode = (BYTE*) ptr;
zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
zc->stage = ZSTDcs_init;
zc->dictID = 0;
zc->loadedDictEnd = 0;
return 0;
} }
zc->seqStore.sequencesStart = (seqDef*)ptr;
ptr = zc->seqStore.sequencesStart + maxNbSeq;
zc->seqStore.llCode = (BYTE*) ptr;
zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq;
zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq;
zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq;
zc->stage = ZSTDcs_init;
zc->dictID = 0;
zc->loadedDictEnd = 0;
return 0;
} }
@@ -313,7 +327,7 @@ size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx)
if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong);
memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem));
ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, 0); ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, srcCCtx->frameContentSize, ZSTDcrp_noMemset);
dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */ dstCCtx->params.fParams.contentSizeFlag = 0; /* content size different from the one set during srcCCtx init */
/* copy tables */ /* copy tables */
@@ -668,8 +682,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* zc,
FSE_CState_t stateOffsetBits; FSE_CState_t stateOffsetBits;
FSE_CState_t stateLitLength; FSE_CState_t stateLitLength;
{ size_t const errorCode = BIT_initCStream(&blockStream, op, oend-op); CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */
if (ERR_isError(errorCode)) return ERROR(dstSize_tooSmall); } /* not enough space remaining */
/* first symbols */ /* first symbols */
FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]);
@@ -1083,7 +1096,7 @@ static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx,
if (ip <= ilimit) { if (ip <= ilimit) {
/* Fill Table */ /* Fill Table */
hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2;
hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base);
/* check immediate repcode */ /* check immediate repcode */
while (ip <= ilimit) { while (ip <= ilimit) {
@@ -2476,8 +2489,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog; unsigned offcodeMaxValue = MaxOff, offcodeLog = OffFSELog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog); CHECK_E (FSE_buildCTable(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
@@ -2485,8 +2497,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); CHECK_E (FSE_buildCTable(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
@@ -2494,8 +2505,7 @@ static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog); CHECK_E(FSE_buildCTable(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
@@ -2520,9 +2530,9 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4);
/* known magic number : dict is parsed for entropy stats and content */ /* known magic number : dict is parsed for entropy stats and content */
{ size_t const eSize_8 = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); { size_t const loadError = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8);
size_t const eSize = eSize_8 + 8; size_t const eSize = loadError + 8;
if (ZSTD_isError(eSize_8)) return eSize_8; if (ZSTD_isError(loadError)) return loadError;
return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize);
} }
} }
@@ -2530,14 +2540,13 @@ static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, si
/*! ZSTD_compressBegin_internal() : /*! ZSTD_compressBegin_internal() :
* @return : 0, or an error code */ * @return : 0, or an error code */
static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* zc, static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, U64 pledgedSrcSize) ZSTD_parameters params, U64 pledgedSrcSize)
{ {
size_t const resetError = ZSTD_resetCCtx_advanced(zc, params, pledgedSrcSize, 1); ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue;
if (ZSTD_isError(resetError)) return resetError; CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp));
return ZSTD_compress_insertDictionary(cctx, dict, dictSize);
return ZSTD_compress_insertDictionary(zc, dict, dictSize);
} }
@@ -2548,9 +2557,7 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx,
ZSTD_parameters params, unsigned long long pledgedSrcSize) ZSTD_parameters params, unsigned long long pledgedSrcSize)
{ {
/* compression parameters verification and optimization */ /* compression parameters verification and optimization */
{ size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize); CHECK_F(ZSTD_checkCParams_advanced(params.cParams, pledgedSrcSize));
if (ZSTD_isError(errorCode)) return errorCode; }
return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize);
} }
@@ -2628,9 +2635,7 @@ static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
size_t const errorCode = ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize); CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize));
if(ZSTD_isError(errorCode)) return errorCode;
return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize);
} }
@@ -2640,8 +2645,7 @@ size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx,
const void* dict,size_t dictSize, const void* dict,size_t dictSize,
ZSTD_parameters params) ZSTD_parameters params)
{ {
size_t const errorCode = ZSTD_checkCParams_advanced(params.cParams, srcSize); CHECK_F(ZSTD_checkCParams_advanced(params.cParams, srcSize));
if (ZSTD_isError(errorCode)) return errorCode;
return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params);
} }
@@ -2677,6 +2681,12 @@ struct ZSTD_CDict_s {
ZSTD_CCtx* refContext; ZSTD_CCtx* refContext;
}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ }; /* typedef'd tp ZSTD_CDict within "zstd.h" */
size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict)
{
if (cdict==NULL) return 0; /* support sizeof on NULL */
return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize;
}
ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem)
{ {
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
@@ -2728,17 +2738,23 @@ size_t ZSTD_freeCDict(ZSTD_CDict* cdict)
} }
} }
size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize)
{
if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext))
else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize));
return 0;
}
/*! ZSTD_compress_usingCDict() : /*! ZSTD_compress_usingCDict() :
* Compression using a digested Dictionary. * Compression using a digested Dictionary.
* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times.
* Note that compression level is decided during dictionary creation */ * Note that compression level is decided during dictionary creation */
ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const ZSTD_CDict* cdict) const ZSTD_CDict* cdict)
{ {
size_t const errorCode = ZSTD_copyCCtx(cctx, cdict->refContext); CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize));
if (ZSTD_isError(errorCode)) return errorCode;
if (cdict->refContext->params.fParams.contentSizeFlag==1) { if (cdict->refContext->params.fParams.contentSizeFlag==1) {
cctx->params.fParams.contentSizeFlag = 1; cctx->params.fParams.contentSizeFlag = 1;
@@ -2757,7 +2773,8 @@ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx,
typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage;
struct ZSTD_CStream_s { struct ZSTD_CStream_s {
ZSTD_CCtx* zc; ZSTD_CCtx* cctx;
ZSTD_CDict* cdict;
char* inBuff; char* inBuff;
size_t inBuffSize; size_t inBuffSize;
size_t inToCompress; size_t inToCompress;
@@ -2790,8 +2807,8 @@ ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem)
if (zcs==NULL) return NULL; if (zcs==NULL) return NULL;
memset(zcs, 0, sizeof(ZSTD_CStream)); memset(zcs, 0, sizeof(ZSTD_CStream));
memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem));
zcs->zc = ZSTD_createCCtx_advanced(customMem); zcs->cctx = ZSTD_createCCtx_advanced(customMem);
if (zcs->zc == NULL) { ZSTD_freeCStream(zcs); return NULL; } if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; }
return zcs; return zcs;
} }
@@ -2799,7 +2816,8 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
{ {
if (zcs==NULL) return 0; /* support free on NULL */ if (zcs==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = zcs->customMem; { ZSTD_customMem const cMem = zcs->customMem;
ZSTD_freeCCtx(zcs->zc); ZSTD_freeCCtx(zcs->cctx);
ZSTD_freeCDict(zcs->cdict);
ZSTD_free(zcs->inBuff, cMem); ZSTD_free(zcs->inBuff, cMem);
ZSTD_free(zcs->outBuff, cMem); ZSTD_free(zcs->outBuff, cMem);
ZSTD_free(zcs, cMem); ZSTD_free(zcs, cMem);
@@ -2813,6 +2831,19 @@ size_t ZSTD_freeCStream(ZSTD_CStream* zcs)
size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; }
size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; }
size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize)
{
CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize));
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->frameEnded = 0;
return 0; /* ready to go */
}
size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
const void* dict, size_t dictSize, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize) ZSTD_parameters params, unsigned long long pledgedSrcSize)
@@ -2834,17 +2865,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs,
if (zcs->outBuff == NULL) return ERROR(memory_allocation); if (zcs->outBuff == NULL) return ERROR(memory_allocation);
} }
{ size_t const errorCode = ZSTD_compressBegin_advanced(zcs->zc, dict, dictSize, params, pledgedSrcSize); ZSTD_freeCDict(zcs->cdict);
if (ZSTD_isError(errorCode)) return errorCode; } zcs->cdict = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem);
if (zcs->cdict == NULL) return ERROR(memory_allocation);
zcs->inToCompress = 0;
zcs->inBuffPos = 0;
zcs->inBuffTarget = zcs->blockSize;
zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0;
zcs->stage = zcss_load;
zcs->checksum = params.fParams.checksumFlag > 0; zcs->checksum = params.fParams.checksumFlag > 0;
zcs->frameEnded = 0;
return 0; /* ready to go */ return ZSTD_resetCStream(zcs, pledgedSrcSize);
} }
size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel)
@@ -2860,7 +2887,8 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel)
size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs)
{ {
return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->zc) + zcs->outBuffSize + zcs->inBuffSize; if (zcs==NULL) return 0; /* support sizeof on NULL */
return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdict) + zcs->outBuffSize + zcs->inBuffSize;
} }
/*====== Compression ======*/ /*====== Compression ======*/
@@ -2911,8 +2939,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs,
else else
cDst = zcs->outBuff, oSize = zcs->outBuffSize; cDst = zcs->outBuff, oSize = zcs->outBuffSize;
cSize = (flush == zsf_end) ? cSize = (flush == zsf_end) ?
ZSTD_compressEnd(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) :
ZSTD_compressContinue(zcs->zc, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize);
if (ZSTD_isError(cSize)) return cSize; if (ZSTD_isError(cSize)) return cSize;
if (flush == zsf_end) zcs->frameEnded = 1; if (flush == zsf_end) zcs->frameEnded = 1;
/* prepare next block */ /* prepare next block */
@@ -2977,8 +3005,8 @@ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
size_t srcSize = 0; size_t srcSize = 0;
size_t sizeWritten = output->size - output->pos; size_t sizeWritten = output->size - output->pos;
size_t const result = ZSTD_compressStream_generic(zcs, size_t const result = ZSTD_compressStream_generic(zcs,
(char*)(output->dst) + output->pos, &sizeWritten, (char*)(output->dst) + output->pos, &sizeWritten,
&srcSize, &srcSize, /* use a valid src address instead of NULL */ &srcSize, &srcSize, /* use a valid src address instead of NULL */
zsf_flush); zsf_flush);
output->pos += sizeWritten; output->pos += sizeWritten;
if (ZSTD_isError(result)) return result; if (ZSTD_isError(result)) return result;
@@ -3006,7 +3034,7 @@ size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output)
/* create epilogue */ /* create epilogue */
zcs->stage = zcss_final; zcs->stage = zcss_final;
zcs->outBuffContentSize = !notEnded ? 0 : zcs->outBuffContentSize = !notEnded ? 0 :
ZSTD_compressEnd(zcs->zc, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */
} }
/* flush epilogue */ /* flush epilogue */

339
C/zstd/zstd_decompress.c
View File

@@ -28,7 +28,6 @@
# define ZSTD_LEGACY_SUPPORT 0 # define ZSTD_LEGACY_SUPPORT 0
#endif #endif
/*! /*!
* MAXWINDOWSIZE_DEFAULT : * MAXWINDOWSIZE_DEFAULT :
* maximum window size accepted by DStream, by default. * maximum window size accepted by DStream, by default.
@@ -57,28 +56,6 @@
#endif #endif
/*-*******************************************************
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/*-************************************* /*-*************************************
* Macros * Macros
***************************************/ ***************************************/
@@ -103,8 +80,12 @@ typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
struct ZSTD_DCtx_s struct ZSTD_DCtx_s
{ {
const FSE_DTable* LLTptr;
const FSE_DTable* MLTptr;
const FSE_DTable* OFTptr;
const HUF_DTable* HUFptr;
FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)];
FSE_DTable OffTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)];
FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)];
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
const void* previousDstEnd; const void* previousDstEnd;
@@ -130,23 +111,27 @@ struct ZSTD_DCtx_s
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ }; /* typedef'd to ZSTD_DCtx within "zstd.h" */
size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return sizeof(*dctx); } size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; return sizeof(ZSTD_DCtx); } /* support sizeof on NULL */
size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); }
size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx)
{ {
dctx->expected = ZSTD_frameHeaderSize_min; dctx->expected = ZSTD_frameHeaderSize_prefix;
dctx->stage = ZSTDds_getFrameHeaderSize; dctx->stage = ZSTDds_getFrameHeaderSize;
dctx->previousDstEnd = NULL; dctx->previousDstEnd = NULL;
dctx->base = NULL; dctx->base = NULL;
dctx->vBase = NULL; dctx->vBase = NULL;
dctx->dictEnd = NULL; dctx->dictEnd = NULL;
dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
dctx->litEntropy = dctx->fseEntropy = 0; dctx->litEntropy = dctx->fseEntropy = 0;
dctx->dictID = 0; dctx->dictID = 0;
MEM_STATIC_ASSERT(sizeof(dctx->rep)==sizeof(repStartValue)); MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue));
memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */
dctx->LLTptr = dctx->LLTable;
dctx->MLTptr = dctx->MLTable;
dctx->OFTptr = dctx->OFTable;
dctx->HUFptr = dctx->hufTable;
return 0; return 0;
} }
@@ -157,7 +142,7 @@ ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem)
if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem;
if (!customMem.customAlloc || !customMem.customFree) return NULL; if (!customMem.customAlloc || !customMem.customFree) return NULL;
dctx = (ZSTD_DCtx*) ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem);
if (!dctx) return NULL; if (!dctx) return NULL;
memcpy(&dctx->customMem, &customMem, sizeof(customMem)); memcpy(&dctx->customMem, &customMem, sizeof(customMem));
ZSTD_decompressBegin(dctx); ZSTD_decompressBegin(dctx);
@@ -182,6 +167,25 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */
} }
static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
{
ZSTD_decompressBegin(dstDCtx); /* init */
dstDCtx->dictEnd = srcDCtx->dictEnd;
dstDCtx->vBase = srcDCtx->vBase;
dstDCtx->base = srcDCtx->base;
dstDCtx->previousDstEnd = srcDCtx->previousDstEnd;
dstDCtx->dictID = srcDCtx->dictID;
dstDCtx->litEntropy = srcDCtx->litEntropy;
dstDCtx->fseEntropy = srcDCtx->fseEntropy;
dstDCtx->LLTptr = srcDCtx->LLTable;
dstDCtx->MLTptr = srcDCtx->MLTable;
dstDCtx->OFTptr = srcDCtx->OFTable;
dstDCtx->HUFptr = srcDCtx->hufTable;
dstDCtx->rep[0] = srcDCtx->rep[0];
dstDCtx->rep[1] = srcDCtx->rep[1];
dstDCtx->rep[2] = srcDCtx->rep[2];
}
/*-************************************************************* /*-*************************************************************
* Decompression section * Decompression section
@@ -190,16 +194,16 @@ void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx)
/* See compression format details in : zstd_compression_format.md */ /* See compression format details in : zstd_compression_format.md */
/** ZSTD_frameHeaderSize() : /** ZSTD_frameHeaderSize() :
* srcSize must be >= ZSTD_frameHeaderSize_min. * srcSize must be >= ZSTD_frameHeaderSize_prefix.
* @return : size of the Frame Header */ * @return : size of the Frame Header */
static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize)
{ {
if (srcSize < ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong);
{ BYTE const fhd = ((const BYTE*)src)[4]; { BYTE const fhd = ((const BYTE*)src)[4];
U32 const dictID= fhd & 3; U32 const dictID= fhd & 3;
U32 const singleSegment = (fhd >> 5) & 1; U32 const singleSegment = (fhd >> 5) & 1;
U32 const fcsId = fhd >> 6; U32 const fcsId = fhd >> 6;
return ZSTD_frameHeaderSize_min + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId]
+ (singleSegment && !fcsId); + (singleSegment && !fcsId);
} }
} }
@@ -214,7 +218,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
{ {
const BYTE* ip = (const BYTE*)src; const BYTE* ip = (const BYTE*)src;
if (srcSize < ZSTD_frameHeaderSize_min) return ZSTD_frameHeaderSize_min; if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix;
if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) {
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) {
if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */
@@ -373,34 +377,31 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
{ {
case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */
/* 2 - 2 - 10 - 10 */ /* 2 - 2 - 10 - 10 */
{ singleStream = !lhlCode; singleStream = !lhlCode;
lhSize = 3; lhSize = 3;
litSize = (lhc >> 4) & 0x3FF; litSize = (lhc >> 4) & 0x3FF;
litCSize = (lhc >> 14) & 0x3FF; litCSize = (lhc >> 14) & 0x3FF;
break; break;
}
case 2: case 2:
/* 2 - 2 - 14 - 14 */ /* 2 - 2 - 14 - 14 */
{ lhSize = 4; lhSize = 4;
litSize = (lhc >> 4) & 0x3FFF; litSize = (lhc >> 4) & 0x3FFF;
litCSize = lhc >> 18; litCSize = lhc >> 18;
break; break;
}
case 3: case 3:
/* 2 - 2 - 18 - 18 */ /* 2 - 2 - 18 - 18 */
{ lhSize = 5; lhSize = 5;
litSize = (lhc >> 4) & 0x3FFFF; litSize = (lhc >> 4) & 0x3FFFF;
litCSize = (lhc >> 22) + (istart[4] << 10); litCSize = (lhc >> 22) + (istart[4] << 10);
break; break;
}
} }
if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected);
if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); if (litCSize + lhSize > srcSize) return ERROR(corruption_detected);
if (HUF_isError((litEncType==set_repeat) ? if (HUF_isError((litEncType==set_repeat) ?
( singleStream ? ( singleStream ?
HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) : HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) :
HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->hufTable) ) : HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) :
( singleStream ? ( singleStream ?
HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) :
HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) ))
@@ -410,6 +411,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH; dctx->litBufSize = ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH;
dctx->litSize = litSize; dctx->litSize = litSize;
dctx->litEntropy = 1; dctx->litEntropy = 1;
if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable;
return litCSize + lhSize; return litCSize + lhSize;
} }
@@ -476,7 +478,6 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
default: default:
return ERROR(corruption_detected); /* impossible */ return ERROR(corruption_detected); /* impossible */
} }
} }
} }
@@ -485,7 +486,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx,
@return : nb bytes read from src, @return : nb bytes read from src,
or an error code if it fails, testable with ZSTD_isError() or an error code if it fails, testable with ZSTD_isError()
*/ */
FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog, static size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e type, U32 max, U32 maxLog,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable) const S16* defaultNorm, U32 defaultLog, U32 flagRepeatTable)
{ {
@@ -515,8 +516,7 @@ FORCE_INLINE size_t ZSTD_buildSeqTable(FSE_DTable* DTable, symbolEncodingType_e
} }
size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr, size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
FSE_DTable* DTableLL, FSE_DTable* DTableML, FSE_DTable* DTableOffb, U32 flagRepeatTable,
const void* src, size_t srcSize) const void* src, size_t srcSize)
{ {
const BYTE* const istart = (const BYTE* const)src; const BYTE* const istart = (const BYTE* const)src;
@@ -546,16 +546,19 @@ size_t ZSTD_decodeSeqHeaders(int* nbSeqPtr,
ip++; ip++;
/* Build DTables */ /* Build DTables */
{ size_t const llhSize = ZSTD_buildSeqTable(DTableLL, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, flagRepeatTable); { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, LLtype, MaxLL, LLFSELog, ip, iend-ip, LL_defaultNorm, LL_defaultNormLog, dctx->fseEntropy);
if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); if (ZSTD_isError(llhSize)) return ERROR(corruption_detected);
if (LLtype != set_repeat) dctx->LLTptr = dctx->LLTable;
ip += llhSize; ip += llhSize;
} }
{ size_t const ofhSize = ZSTD_buildSeqTable(DTableOffb, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, flagRepeatTable); { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, OFtype, MaxOff, OffFSELog, ip, iend-ip, OF_defaultNorm, OF_defaultNormLog, dctx->fseEntropy);
if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected);
if (OFtype != set_repeat) dctx->OFTptr = dctx->OFTable;
ip += ofhSize; ip += ofhSize;
} }
{ size_t const mlhSize = ZSTD_buildSeqTable(DTableML, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, flagRepeatTable); { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, MLtype, MaxML, MLFSELog, ip, iend-ip, ML_defaultNorm, ML_defaultNormLog, dctx->fseEntropy);
if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected);
if (MLtype != set_repeat) dctx->MLTptr = dctx->MLTable;
ip += mlhSize; ip += mlhSize;
} } } }
@@ -738,16 +741,13 @@ static size_t ZSTD_decompressSequences(
const BYTE* litPtr = dctx->litPtr; const BYTE* litPtr = dctx->litPtr;
const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH; const BYTE* const litLimit_w = litPtr + dctx->litBufSize - WILDCOPY_OVERLENGTH;
const BYTE* const litEnd = litPtr + dctx->litSize; const BYTE* const litEnd = litPtr + dctx->litSize;
FSE_DTable* DTableLL = dctx->LLTable;
FSE_DTable* DTableML = dctx->MLTable;
FSE_DTable* DTableOffb = dctx->OffTable;
const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
int nbSeq; int nbSeq;
/* Build Decoding Tables */ /* Build Decoding Tables */
{ size_t const seqHSize = ZSTD_decodeSeqHeaders(&nbSeq, DTableLL, DTableML, DTableOffb, dctx->fseEntropy, ip, seqSize); { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize);
if (ZSTD_isError(seqHSize)) return seqHSize; if (ZSTD_isError(seqHSize)) return seqHSize;
ip += seqHSize; ip += seqHSize;
} }
@@ -757,11 +757,10 @@ static size_t ZSTD_decompressSequences(
seqState_t seqState; seqState_t seqState;
dctx->fseEntropy = 1; dctx->fseEntropy = 1;
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; } { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->rep[i]; }
{ size_t const errorCode = BIT_initDStream(&(seqState.DStream), ip, iend-ip); CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected);
if (ERR_isError(errorCode)) return ERROR(corruption_detected); } FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr);
FSE_initDState(&(seqState.stateLL), &(seqState.DStream), DTableLL); FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr);
FSE_initDState(&(seqState.stateOffb), &(seqState.DStream), DTableOffb); FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr);
FSE_initDState(&(seqState.stateML), &(seqState.DStream), DTableML);
for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) {
nbSeq--; nbSeq--;
@@ -863,12 +862,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
/* Frame Header */ /* Frame Header */
{ size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
size_t result;
if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize;
if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong);
result = ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize); CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize));
if (ZSTD_isError(result)) return result;
ip += frameHeaderSize; remainingSize -= frameHeaderSize; ip += frameHeaderSize; remainingSize -= frameHeaderSize;
} }
@@ -921,25 +918,10 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx,
} }
/*! ZSTD_decompress_usingPreparedDCtx() :
* Same as ZSTD_decompress_usingDict, but using a reference context `preparedDCtx`, where dictionary has been loaded.
* It avoids reloading the dictionary each time.
* `preparedDCtx` must have been properly initialized using ZSTD_decompressBegin_usingDict().
* Requires 2 contexts : 1 for reference (preparedDCtx), which will not be modified, and 1 to run the decompression operation (dctx) */
size_t ZSTD_decompress_usingPreparedDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* refDCtx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize)
{
ZSTD_copyDCtx(dctx, refDCtx);
ZSTD_checkContinuity(dctx, dst);
return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
}
size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const void* dict, size_t dictSize) const void* dict, size_t dictSize)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize);
@@ -1013,32 +995,30 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
switch (dctx->stage) switch (dctx->stage)
{ {
case ZSTDds_getFrameHeaderSize : case ZSTDds_getFrameHeaderSize :
if (srcSize != ZSTD_frameHeaderSize_min) return ERROR(srcSize_wrong); /* impossible */ if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */
if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_min; /* magic number + skippable frame length */ dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */
dctx->stage = ZSTDds_decodeSkippableHeader; dctx->stage = ZSTDds_decodeSkippableHeader;
return 0; return 0;
} }
dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_min); dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix);
if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize;
memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_min); memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix);
if (dctx->headerSize > ZSTD_frameHeaderSize_min) { if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) {
dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_min; dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix;
dctx->stage = ZSTDds_decodeFrameHeader; dctx->stage = ZSTDds_decodeFrameHeader;
return 0; return 0;
} }
dctx->expected = 0; /* not necessary to copy more */ dctx->expected = 0; /* not necessary to copy more */
case ZSTDds_decodeFrameHeader: case ZSTDds_decodeFrameHeader:
{ size_t result; memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize));
result = ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize); dctx->expected = ZSTD_blockHeaderSize;
if (ZSTD_isError(result)) return result; dctx->stage = ZSTDds_decodeBlockHeader;
dctx->expected = ZSTD_blockHeaderSize; return 0;
dctx->stage = ZSTDds_decodeBlockHeader;
return 0;
}
case ZSTDds_decodeBlockHeader: case ZSTDds_decodeBlockHeader:
{ blockProperties_t bp; { blockProperties_t bp;
size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp);
@@ -1110,7 +1090,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c
return 0; return 0;
} }
case ZSTDds_decodeSkippableHeader: case ZSTDds_decodeSkippableHeader:
{ memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_min, src, dctx->expected); { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected);
dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); dctx->expected = MEM_readLE32(dctx->headerBuffer + 4);
dctx->stage = ZSTDds_skipFrame; dctx->stage = ZSTDds_skipFrame;
return 0; return 0;
@@ -1149,8 +1129,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog; U32 offcodeMaxValue=MaxOff, offcodeLog=OffFSELog;
size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->OffTable, offcodeNCount, offcodeMaxValue, offcodeLog); CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += offcodeHeaderSize; dictPtr += offcodeHeaderSize;
} }
@@ -1158,8 +1137,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog; unsigned matchlengthMaxValue = MaxML, matchlengthLog = MLFSELog;
size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog); CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += matchlengthHeaderSize; dictPtr += matchlengthHeaderSize;
} }
@@ -1167,8 +1145,7 @@ static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t c
unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog; unsigned litlengthMaxValue = MaxLL, litlengthLog = LLFSELog;
size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr);
if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted);
{ size_t const errorCode = FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog); CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted);
if (FSE_isError(errorCode)) return ERROR(dictionary_corrupted); }
dictPtr += litlengthHeaderSize; dictPtr += litlengthHeaderSize;
} }
@@ -1204,21 +1181,16 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict
return ZSTD_refDictContent(dctx, dict, dictSize); return ZSTD_refDictContent(dctx, dict, dictSize);
} }
size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize)
{ {
{ size_t const errorCode = ZSTD_decompressBegin(dctx); CHECK_F(ZSTD_decompressBegin(dctx));
if (ZSTD_isError(errorCode)) return errorCode; } if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted);
if (dict && dictSize) {
size_t const errorCode = ZSTD_decompress_insertDictionary(dctx, dict, dictSize);
if (ZSTD_isError(errorCode)) return ERROR(dictionary_corrupted);
}
return 0; return 0;
} }
/* ====== ZSTD_DDict ====== */
struct ZSTD_DDict_s { struct ZSTD_DDict_s {
void* dict; void* dict;
size_t dictSize; size_t dictSize;
@@ -1277,20 +1249,27 @@ size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
} }
} }
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize;
}
/*! ZSTD_decompress_usingDDict() : /*! ZSTD_decompress_usingDDict() :
* Decompression using a pre-digested Dictionary * Decompression using a pre-digested Dictionary
* Use dictionary without significant overhead. */ * Use dictionary without significant overhead. */
ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity, void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const void* src, size_t srcSize,
const ZSTD_DDict* ddict) const ZSTD_DDict* ddict)
{ {
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1)
if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize);
#endif #endif
return ZSTD_decompress_usingPreparedDCtx(dctx, ddict->refContext, ZSTD_refDCtx(dctx, ddict->refContext);
dst, dstCapacity, ZSTD_checkContinuity(dctx, dst);
src, srcSize); return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize);
} }
@@ -1303,7 +1282,8 @@ typedef enum { zdss_init, zdss_loadHeader,
/* *** Resource management *** */ /* *** Resource management *** */
struct ZSTD_DStream_s { struct ZSTD_DStream_s {
ZSTD_DCtx* zd; ZSTD_DCtx* dctx;
ZSTD_DDict* ddict;
ZSTD_frameParams fParams; ZSTD_frameParams fParams;
ZSTD_dStreamStage stage; ZSTD_dStreamStage stage;
char* inBuff; char* inBuff;
@@ -1315,15 +1295,13 @@ struct ZSTD_DStream_s {
size_t outStart; size_t outStart;
size_t outEnd; size_t outEnd;
size_t blockSize; size_t blockSize;
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */
size_t lhSize; size_t lhSize;
ZSTD_customMem customMem; ZSTD_customMem customMem;
void* dictContent;
size_t dictSize;
const void* dictSource;
void* legacyContext; void* legacyContext;
U32 previousLegacyVersion; U32 previousLegacyVersion;
U32 legacyVersion; U32 legacyVersion;
U32 hostageByte;
}; /* typedef'd to ZSTD_DStream within "zstd.h" */ }; /* typedef'd to ZSTD_DStream within "zstd.h" */
@@ -1343,8 +1321,8 @@ ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem)
if (zds==NULL) return NULL; if (zds==NULL) return NULL;
memset(zds, 0, sizeof(ZSTD_DStream)); memset(zds, 0, sizeof(ZSTD_DStream));
memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem));
zds->zd = ZSTD_createDCtx_advanced(customMem); zds->dctx = ZSTD_createDCtx_advanced(customMem);
if (zds->zd == NULL) { ZSTD_freeDStream(zds); return NULL; } if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; }
zds->stage = zdss_init; zds->stage = zdss_init;
zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT;
return zds; return zds;
@@ -1354,10 +1332,10 @@ size_t ZSTD_freeDStream(ZSTD_DStream* zds)
{ {
if (zds==NULL) return 0; /* support free on null */ if (zds==NULL) return 0; /* support free on null */
{ ZSTD_customMem const cMem = zds->customMem; { ZSTD_customMem const cMem = zds->customMem;
ZSTD_freeDCtx(zds->zd); ZSTD_freeDCtx(zds->dctx);
ZSTD_freeDDict(zds->ddict);
ZSTD_free(zds->inBuff, cMem); ZSTD_free(zds->inBuff, cMem);
ZSTD_free(zds->outBuff, cMem); ZSTD_free(zds->outBuff, cMem);
ZSTD_free(zds->dictContent, cMem);
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1)
if (zds->legacyContext) if (zds->legacyContext)
ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion);
@@ -1377,17 +1355,12 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di
{ {
zds->stage = zdss_loadHeader; zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
if ((dict != zds->dictSource) | (dictSize != zds->dictSize)) { /* new dictionary */ ZSTD_freeDDict(zds->ddict);
if (dictSize > zds->dictSize) { zds->ddict = ZSTD_createDDict(dict, dictSize);
ZSTD_free(zds->dictContent, zds->customMem); if (zds->ddict == NULL) return ERROR(memory_allocation);
zds->dictContent = ZSTD_malloc(dictSize, zds->customMem);
if (zds->dictContent == NULL) return ERROR(memory_allocation);
}
memcpy(zds->dictContent, dict, dictSize);
zds->dictSize = dictSize;
}
zds->legacyVersion = 0; zds->legacyVersion = 0;
return 0; zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix;
} }
size_t ZSTD_initDStream(ZSTD_DStream* zds) size_t ZSTD_initDStream(ZSTD_DStream* zds)
@@ -1395,6 +1368,15 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds)
return ZSTD_initDStream_usingDict(zds, NULL, 0); return ZSTD_initDStream_usingDict(zds, NULL, 0);
} }
size_t ZSTD_resetDStream(ZSTD_DStream* zds)
{
zds->stage = zdss_loadHeader;
zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0;
zds->legacyVersion = 0;
zds->hostageByte = 0;
return ZSTD_frameHeaderSize_prefix;
}
size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
ZSTD_DStreamParameter_e paramType, unsigned paramValue) ZSTD_DStreamParameter_e paramType, unsigned paramValue)
{ {
@@ -1409,11 +1391,12 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds)
{ {
return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->zd) + zds->inBuffSize + zds->outBuffSize; if (zds==NULL) return 0; /* support sizeof on NULL */
return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddict) + zds->inBuffSize + zds->outBuffSize;
} }
/* *** Decompression *** */ /* ***** Decompression ***** */
MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize)
{ {
@@ -1450,10 +1433,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
{ U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart);
if (legacyVersion) { if (legacyVersion) {
size_t initResult; CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion,
initResult = ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, zds->ddict->dict, zds->ddict->dictSize));
zds->dictContent, zds->dictSize);
if (ZSTD_isError(initResult)) return initResult;
zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; zds->legacyVersion = zds->previousLegacyVersion = legacyVersion;
return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input);
} else { } else {
@@ -1468,27 +1449,24 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip);
zds->lhSize += iend-ip; zds->lhSize += iend-ip;
input->pos = input->size; input->pos = input->size;
return (hSize - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */
} }
memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad;
break; break;
} } } }
/* Consume header */ /* Consume header */
ZSTD_decompressBegin_usingDict(zds->zd, zds->dictContent, zds->dictSize); ZSTD_refDCtx(zds->dctx, zds->ddict->refContext);
{ size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->zd); /* == ZSTD_frameHeaderSize_min */ { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */
size_t const h1Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer, h1Size); CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size));
if (ZSTD_isError(h1Result)) return h1Result; /* should not happen : already checked */ { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (h1Size < zds->lhSize) { /* long header */ CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size));
size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->zd);
size_t const h2Result = ZSTD_decompressContinue(zds->zd, NULL, 0, zds->headerBuffer+h1Size, h2Size);
if (ZSTD_isError(h2Result)) return h2Result;
} } } }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported); if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
/* Frame header instruct buffer sizes */ /* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);
size_t const neededOutSize = zds->fParams.windowSize + blockSize; size_t const neededOutSize = zds->fParams.windowSize + blockSize;
zds->blockSize = blockSize; zds->blockSize = blockSize;
@@ -1508,21 +1486,21 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* pass-through */ /* pass-through */
case zdss_read: case zdss_read:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (neededInSize==0) { /* end of frame */ if (neededInSize==0) { /* end of frame */
zds->stage = zdss_init; zds->stage = zdss_init;
someMoreWork = 0; someMoreWork = 0;
break; break;
} }
if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */
const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd, size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart),
ip, neededInSize); ip, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
ip += neededInSize; ip += neededInSize;
if (!decodedSize && !isSkipFrame) break; /* this was just a header */ if (!decodedSize && !isSkipFrame) break; /* this was just a header */
zds->outEnd = zds->outStart + decodedSize; zds->outEnd = zds->outStart + decodedSize;
zds->stage = zdss_flush; zds->stage = zdss_flush;
break; break;
} }
@@ -1532,7 +1510,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
case zdss_load: case zdss_load:
{ size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->zd); { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx);
size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */
size_t loadedSize; size_t loadedSize;
if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */
@@ -1542,8 +1520,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */
/* decode loaded input */ /* decode loaded input */
{ const int isSkipFrame = ZSTD_isSkipFrame(zds->zd); { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx);
size_t const decodedSize = ZSTD_decompressContinue(zds->zd, size_t const decodedSize = ZSTD_decompressContinue(zds->dctx,
zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart,
zds->inBuff, neededInSize); zds->inBuff, neededInSize);
if (ZSTD_isError(decodedSize)) return decodedSize; if (ZSTD_isError(decodedSize)) return decodedSize;
@@ -1565,7 +1543,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
zds->outStart = zds->outEnd = 0; zds->outStart = zds->outEnd = 0;
break; break;
} }
/* cannot flush everything */ /* cannot complete flush */
someMoreWork = 0; someMoreWork = 0;
break; break;
} }
@@ -1575,9 +1553,22 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
/* result */ /* result */
input->pos += (size_t)(ip-istart); input->pos += (size_t)(ip-istart);
output->pos += (size_t)(op-ostart); output->pos += (size_t)(op-ostart);
{ size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->zd); { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx);
if (!nextSrcSizeHint) return (zds->outEnd != zds->outStart); /* return 0 only if fully flushed too */ if (!nextSrcSizeHint) { /* frame fully decoded */
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->zd) == ZSTDnit_block); if (zds->outEnd == zds->outStart) { /* output fully flushed */
if (zds->hostageByte) {
if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */
input->pos++; /* release hostage */
}
return 0;
}
if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */
input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */
zds->hostageByte=1;
}
return 1;
}
nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */
if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */
nextSrcSizeHint -= zds->inPos; /* already loaded*/ nextSrcSizeHint -= zds->inPos; /* already loaded*/
return nextSrcSizeHint; return nextSrcSizeHint;

26
C/zstd/zstd_internal.h
View File

@@ -10,6 +10,28 @@
#ifndef ZSTD_CCOMMON_H_MODULE #ifndef ZSTD_CCOMMON_H_MODULE
#define ZSTD_CCOMMON_H_MODULE #define ZSTD_CCOMMON_H_MODULE
/*-*******************************************************
* Compiler specifics
*********************************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
#else
# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/*-************************************* /*-*************************************
* Dependencies * Dependencies
***************************************/ ***************************************/
@@ -20,10 +42,12 @@
/*-************************************* /*-*************************************
* Common macros * shared macros
***************************************/ ***************************************/
#define MIN(a,b) ((a)<(b) ? (a) : (b)) #define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b)) #define MAX(a,b) ((a)>(b) ? (a) : (b))
#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */
#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */
/*-************************************* /*-*************************************

190
C/zstd/zstd_v05.c
View File

@@ -10,6 +10,7 @@
/*- Dependencies -*/ /*- Dependencies -*/
#include "zstd_v05.h" #include "zstd_v05.h"
#include "error_private.h"
/* ****************************************************************** /* ******************************************************************
@@ -254,79 +255,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif /* MEM_H_MODULE */ #endif /* MEM_H_MODULE */
/* ******************************************************************
Error codes list
Copyright (C) 2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/zstd
****************************************************************** */
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* error codes list
******************************************/
typedef enum {
ZSTDv05_error_no_error,
ZSTDv05_error_GENERIC,
ZSTDv05_error_prefix_unknown,
ZSTDv05_error_frameParameter_unsupported,
ZSTDv05_error_frameParameter_unsupportedBy32bits,
ZSTDv05_error_init_missing,
ZSTDv05_error_memory_allocation,
ZSTDv05_error_stage_wrong,
ZSTDv05_error_dstSize_tooSmall,
ZSTDv05_error_srcSize_wrong,
ZSTDv05_error_corruption_detected,
ZSTDv05_error_tableLog_tooLarge,
ZSTDv05_error_maxSymbolValue_tooLarge,
ZSTDv05_error_maxSymbolValue_tooSmall,
ZSTDv05_error_dictionary_corrupted,
ZSTDv05_error_maxCode
} ZSTDv05_ErrorCode;
/* note : functions provide error codes in reverse negative order,
so compare with (size_t)(0-enum) */
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */
/* /*
zstd - standard compression library zstd - standard compression library
Header File for static linking only Header File for static linking only
@@ -470,119 +398,6 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity
#endif /* ZSTDv05_STATIC_H */ #endif /* ZSTDv05_STATIC_H */
/* ******************************************************************
Error codes and messages
Copyright (C) 2013-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Source repository : https://github.com/Cyan4973/zstd
****************************************************************** */
/* Note : this module is expected to remain private, do not expose it */
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Compiler-specific
******************************************/
#if defined(__GNUC__)
# define ERR_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
# define ERR_STATIC static __inline
#else
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Customization
******************************************/
typedef ZSTDv05_ErrorCode ERR_enum;
#define PREFIX(name) ZSTDv05_error_##name
/*-****************************************
* Error codes handling
******************************************/
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define ERROR(name) (size_t)-PREFIX(name)
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getError(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
/*-****************************************
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
static const char* notErrorCode = "Unspecified error code";
switch( ERR_getError(code) )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max possible Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(maxCode):
default: return notErrorCode; /* should be impossible, due to ERR_getError() */
}
}
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */
/* /*
zstd_internal - common functions to include zstd_internal - common functions to include
Header File for include Header File for include
@@ -2883,9 +2698,6 @@ static void ZSTDv05_copy4(void* dst, const void* src) { memcpy(dst, src, 4); }
* tells if a return value is an error code */ * tells if a return value is an error code */
unsigned ZSTDv05_isError(size_t code) { return ERR_isError(code); } unsigned ZSTDv05_isError(size_t code) { return ERR_isError(code); }
/*! ZSTDv05_getError() :
* convert a `size_t` function result into a proper ZSTDv05_errorCode enum */
ZSTDv05_ErrorCode ZSTDv05_getError(size_t code) { return ERR_getError(code); }
/*! ZSTDv05_getErrorName() : /*! ZSTDv05_getErrorName() :
* provides error code string (useful for debugging) */ * provides error code string (useful for debugging) */

205
C/zstd/zstd_v06.c
View File

@@ -13,6 +13,7 @@
#include <stddef.h> /* size_t, ptrdiff_t */ #include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */ #include <string.h> /* memcpy */
#include <stdlib.h> /* malloc, free, qsort */ #include <stdlib.h> /* malloc, free, qsort */
#include "error_private.h"
@@ -273,77 +274,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif /* MEM_H_MODULE */ #endif /* MEM_H_MODULE */
/* ******************************************************************
Error codes list
Copyright (C) 2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* error codes list
******************************************/
typedef enum {
ZSTDv06_error_no_error,
ZSTDv06_error_GENERIC,
ZSTDv06_error_prefix_unknown,
ZSTDv06_error_frameParameter_unsupported,
ZSTDv06_error_frameParameter_unsupportedBy32bits,
ZSTDv06_error_compressionParameter_unsupported,
ZSTDv06_error_init_missing,
ZSTDv06_error_memory_allocation,
ZSTDv06_error_stage_wrong,
ZSTDv06_error_dstSize_tooSmall,
ZSTDv06_error_srcSize_wrong,
ZSTDv06_error_corruption_detected,
ZSTDv06_error_tableLog_tooLarge,
ZSTDv06_error_maxSymbolValue_tooLarge,
ZSTDv06_error_maxSymbolValue_tooSmall,
ZSTDv06_error_dictionary_corrupted,
ZSTDv06_error_maxCode
} ZSTDv06_ErrorCode;
/* note : compare with size_t function results using ZSTDv06_getError() */
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */
/* /*
zstd - standard compression library zstd - standard compression library
Header File for static linking only Header File for static linking only
@@ -469,137 +399,12 @@ ZSTDLIB_API size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx);
ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); ZSTDLIB_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
/*-*************************************
* Error management
***************************************/
/*! ZSTDv06_getErrorCode() :
convert a `size_t` function result into a `ZSTDv06_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTDLIB_API ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t functionResult);
ZSTDLIB_API const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code);
#if defined (__cplusplus) #if defined (__cplusplus)
} }
#endif #endif
#endif /* ZSTDv06_STATIC_H */ #endif /* ZSTDv06_STATIC_H */
/* ******************************************************************
Error codes and messages
Copyright (C) 2013-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
/* Note : this module is expected to remain private, do not expose it */
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Compiler-specific
******************************************/
#if defined(__GNUC__)
# define ERR_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
# define ERR_STATIC static __inline
#else
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Customization (error_public.h)
******************************************/
typedef ZSTDv06_ErrorCode ERR_enum;
#define PREFIX(name) ZSTDv06_error_##name
/*-****************************************
* Error codes handling
******************************************/
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define ERROR(name) ((size_t)-PREFIX(name))
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
/*-****************************************
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
{
static const char* notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(maxCode):
default: return notErrorCode;
}
}
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */
/* /*
zstd_internal - common functions to include zstd_internal - common functions to include
Header File for include Header File for include
@@ -2986,14 +2791,6 @@ unsigned ZSTDv06_isError(size_t code) { return ERR_isError(code); }
* provides error code string from function result (useful for debugging) */ * provides error code string from function result (useful for debugging) */
const char* ZSTDv06_getErrorName(size_t code) { return ERR_getErrorName(code); } const char* ZSTDv06_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*! ZSTDv06_getError() :
* convert a `size_t` function result into a proper ZSTDv06_errorCode enum */
ZSTDv06_ErrorCode ZSTDv06_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! ZSTDv06_getErrorString() :
* provides error code string from enum */
const char* ZSTDv06_getErrorString(ZSTDv06_ErrorCode code) { return ERR_getErrorName(code); }
/* ************************************************************** /* **************************************************************
* ZBUFF Error Management * ZBUFF Error Management

206
C/zstd/zstd_v07.c
View File

@@ -21,6 +21,8 @@
#define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */ #define HUFv07_STATIC_LINKING_ONLY /* HUFv07_TABLELOG_ABSOLUTEMAX */
#define ZSTDv07_STATIC_LINKING_ONLY #define ZSTDv07_STATIC_LINKING_ONLY
#include "error_private.h"
#ifdef ZSTDv07_STATIC_LINKING_ONLY #ifdef ZSTDv07_STATIC_LINKING_ONLY
@@ -427,203 +429,6 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
#endif #endif
#endif /* MEM_H_MODULE */ #endif /* MEM_H_MODULE */
/* ******************************************************************
Error codes list
Copyright (C) 2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
#ifndef ERROR_PUBLIC_H_MODULE
#define ERROR_PUBLIC_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* error codes list
******************************************/
typedef enum {
ZSTDv07_error_no_error,
ZSTDv07_error_GENERIC,
ZSTDv07_error_prefix_unknown,
ZSTDv07_error_frameParameter_unsupported,
ZSTDv07_error_frameParameter_unsupportedBy32bits,
ZSTDv07_error_compressionParameter_unsupported,
ZSTDv07_error_init_missing,
ZSTDv07_error_memory_allocation,
ZSTDv07_error_stage_wrong,
ZSTDv07_error_dstSize_tooSmall,
ZSTDv07_error_srcSize_wrong,
ZSTDv07_error_corruption_detected,
ZSTDv07_error_checksum_wrong,
ZSTDv07_error_tableLog_tooLarge,
ZSTDv07_error_maxSymbolValue_tooLarge,
ZSTDv07_error_maxSymbolValue_tooSmall,
ZSTDv07_error_dictionary_corrupted,
ZSTDv07_error_dictionary_wrong,
ZSTDv07_error_maxCode
} ZSTDv07_ErrorCode;
/*! ZSTDv07_getErrorCode() :
convert a `size_t` function result into a `ZSTDv07_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t functionResult);
const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code);
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_PUBLIC_H_MODULE */
/* ******************************************************************
Error codes and messages
Copyright (C) 2013-2016, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- Homepage : http://www.zstd.net
****************************************************************** */
/* Note : this module is expected to remain private, do not expose it */
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Compiler-specific
******************************************/
#if defined(__GNUC__)
# define ERR_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
# define ERR_STATIC static __inline
#else
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Customization (error_public.h)
******************************************/
typedef ZSTDv07_ErrorCode ERR_enum;
#define PREFIX(name) ZSTDv07_error_##name
/*-****************************************
* Error codes handling
******************************************/
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define ERROR(name) ((size_t)-PREFIX(name))
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
/*-****************************************
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
{
static const char* notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
}
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
return ERR_getErrorString(ERR_getErrorCode(code));
}
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */
/* ****************************************************************** /* ******************************************************************
bitstream bitstream
Part of FSE library Part of FSE library
@@ -2882,13 +2687,6 @@ unsigned ZSTDv07_isError(size_t code) { return ERR_isError(code); }
* provides error code string from function result (useful for debugging) */ * provides error code string from function result (useful for debugging) */
const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); } const char* ZSTDv07_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*! ZSTDv07_getError() :
* convert a `size_t` function result into a proper ZSTDv07_errorCode enum */
ZSTDv07_ErrorCode ZSTDv07_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! ZSTDv07_getErrorString() :
* provides error code string from enum */
const char* ZSTDv07_getErrorString(ZSTDv07_ErrorCode code) { return ERR_getErrorName(code); }
/* ************************************************************** /* **************************************************************

0
CPP/7zip/Archive/Icons/zst.ico Normal file
View File

367
CPP/7zip/Archive/ZstdHandler.cpp Normal file
View File

@@ -0,0 +1,367 @@
// ZstdHandler.cpp
#include "StdAfx.h"
#include "../../../C/CpuArch.h"
#include "../../Common/ComTry.h"
#include "../../Common/Defs.h"
#include "../Common/ProgressUtils.h"
#include "../Common/RegisterArc.h"
#include "../Common/StreamUtils.h"
#include "../Compress/ZstdDecoder.h"
#include "../Compress/ZstdEncoder.h"
#include "../Compress/CopyCoder.h"
#include "Common/DummyOutStream.h"
#include "Common/HandlerOut.h"
#include <stdio.h>
using namespace NWindows;
namespace NArchive {
namespace NZSTD {
class CHandler:
public IInArchive,
public IArchiveOpenSeq,
public IOutArchive,
public ISetProperties,
public CMyUnknownImp
{
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
bool _isArc;
bool _dataAfterEnd;
bool _needMoreInput;
bool _packSize_Defined;
bool _unpackSize_Defined;
UInt64 _packSize;
UInt64 _unpackSize;
UInt64 _numStreams;
UInt64 _numBlocks;
CSingleMethodProps _props;
public:
MY_UNKNOWN_IMP4(
IInArchive,
IArchiveOpenSeq,
IOutArchive,
ISetProperties)
INTERFACE_IInArchive(;)
INTERFACE_IOutArchive(;)
STDMETHOD(OpenSeq)(ISequentialInStream *stream);
STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
CHandler() { }
};
static const Byte kProps[] =
{
kpidSize,
kpidPackSize
};
static const Byte kArcProps[] =
{
kpidNumStreams,
kpidNumBlocks
};
IMP_IInArchive_Props
IMP_IInArchive_ArcProps
STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/)
{
return S_OK;
}
STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
{
*numItems = 1;
return S_OK;
}
STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
{
NCOM::CPropVariant prop;
switch (propID)
{
case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
}
prop.Detach(value);
return S_OK;
}
static const unsigned kSignatureCheckSize = 4;
API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size)
{
if (size < 4)
return k_IsArc_Res_NEED_MORE;
// 0xFD2FB525 = 0.5
// ..
// 0xFD2FB528 = 0.8
UInt32 magic = GetUi32(p);
if (magic >= 0xFD2FB525 && magic <= 0xFD2FB528)
return k_IsArc_Res_YES;
return k_IsArc_Res_NO;
}
}
STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)
{
COM_TRY_BEGIN
Close();
{
Byte buf[kSignatureCheckSize];
RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize));
if (IsArc_zstd(buf, kSignatureCheckSize) == k_IsArc_Res_NO)
return S_FALSE;
_isArc = true;
_stream = stream;
_seqStream = stream;
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
}
return S_OK;
COM_TRY_END
}
STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
{
Close();
_isArc = true;
_seqStream = stream;
return S_OK;
}
STDMETHODIMP CHandler::Close()
{
_isArc = false;
_dataAfterEnd = false;
_needMoreInput = false;
_packSize_Defined = false;
_unpackSize_Defined = false;
_packSize = 0;
_seqStream.Release();
_stream.Release();
return S_OK;
}
STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
Int32 testMode, IArchiveExtractCallback *extractCallback)
{
COM_TRY_BEGIN
if (numItems == 0)
return S_OK;
if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
return E_INVALIDARG;
if (_packSize_Defined)
extractCallback->SetTotal(_packSize);
CMyComPtr<ISequentialOutStream> realOutStream;
Int32 askMode = testMode ?
NExtract::NAskMode::kTest :
NExtract::NAskMode::kExtract;
RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
if (!testMode && !realOutStream)
return S_OK;
extractCallback->PrepareOperation(askMode);
Int32 opRes;
{
NCompress::NZSTD::CDecoder *decoderSpec = new NCompress::NZSTD::CDecoder;
CMyComPtr<ICompressCoder> decoder = decoderSpec;
decoderSpec->SetInStream(_seqStream);
CDummyOutStream *outStreamSpec = new CDummyOutStream;
CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
outStreamSpec->SetStream(realOutStream);
outStreamSpec->Init();
realOutStream.Release();
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(extractCallback, true);
UInt64 packSize = 0;
UInt64 unpackedSize = 0;
HRESULT result = S_OK;
for (;;)
{
lps->InSize = packSize;
lps->OutSize = unpackedSize;
RINOK(lps->SetCur());
result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
UInt64 streamSize = decoderSpec->GetInputProcessedSize();
printf("streamsize=%d packsize=%d unpackedSize=%d\n",
streamSize, packSize, unpackedSize);
fflush(stdout);
if (result != S_FALSE && result != S_OK)
return result;
if (unpackedSize == 0)
break;
if (streamSize == packSize)
{
// no new bytes in input stream, So it's good end of archive.
result = S_OK;
break;
}
if (packSize > streamSize)
return E_FAIL;
if (result != S_OK)
break;
}
decoderSpec->ReleaseInStream();
outStream.Release();
if (!_isArc)
opRes = NExtract::NOperationResult::kIsNotArc;
else if (_needMoreInput)
opRes = NExtract::NOperationResult::kUnexpectedEnd;
else if (_dataAfterEnd)
opRes = NExtract::NOperationResult::kDataAfterEnd;
else if (result == S_FALSE)
opRes = NExtract::NOperationResult::kDataError;
else if (result == S_OK)
opRes = NExtract::NOperationResult::kOK;
else
return result;
}
return extractCallback->SetOperationResult(opRes);
COM_TRY_END
}
static HRESULT UpdateArchive(
UInt64 unpackSize,
ISequentialOutStream *outStream,
const CProps &props,
IArchiveUpdateCallback *updateCallback)
{
RINOK(updateCallback->SetTotal(unpackSize));
CMyComPtr<ISequentialInStream> fileInStream;
RINOK(updateCallback->GetStream(0, &fileInStream));
CLocalProgress *localProgressSpec = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
localProgressSpec->Init(updateCallback, true);
NCompress::NZSTD::CEncoder *encoderSpec = new NCompress::NZSTD::CEncoder;
CMyComPtr<ICompressCoder> encoder = encoderSpec;
RINOK(props.SetCoderProps(encoderSpec, NULL));
RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));
return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
}
STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
{
*type = NFileTimeType::kUnix;
return S_OK;
}
STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
IArchiveUpdateCallback *updateCallback)
{
COM_TRY_BEGIN
if (numItems != 1)
return E_INVALIDARG;
Int32 newData, newProps;
UInt32 indexInArchive;
if (!updateCallback)
return E_FAIL;
RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive));
if ((newProps))
{
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop));
if (prop.vt != VT_EMPTY)
if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
return E_INVALIDARG;
}
}
if ((newData))
{
UInt64 size;
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
if (prop.vt != VT_UI8)
return E_INVALIDARG;
size = prop.uhVal.QuadPart;
}
return UpdateArchive(size, outStream, _props, updateCallback);
}
if (indexInArchive != 0)
return E_INVALIDARG;
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(updateCallback, true);
CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
if (opCallback)
{
RINOK(opCallback->ReportOperation(
NEventIndexType::kInArcIndex, 0,
NUpdateNotifyOp::kReplicate))
}
if (_stream)
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
return NCompress::CopyStream(_stream, outStream, progress);
COM_TRY_END
}
STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
{
return _props.SetProperties(names, values, numProps);
}
static const Byte k_Signature[] = "0xFD2FB525 .. 0xFD2FB528";
REGISTER_ARC_IO(
"zstd", "zst tzstd", "* .tar", 0x0e,
k_Signature,
0,
NArcInfoFlags::kKeepName,
IsArc_zstd)
}}

1
CPP/7zip/Bundles/Format7zF/Arc.mak
View File

@@ -90,6 +90,7 @@ AR_OBJS = \
$O\XarHandler.obj \ $O\XarHandler.obj \
$O\XzHandler.obj \ $O\XzHandler.obj \
$O\ZHandler.obj \ $O\ZHandler.obj \
$O\ZstdHandler.obj \
AR_COMMON_OBJS = \ AR_COMMON_OBJS = \
$O\CoderMixer2.obj \ $O\CoderMixer2.obj \

4
CPP/7zip/Bundles/Format7zF/resource.rc
View File

@@ -28,10 +28,10 @@ MY_VERSION_INFO_DLL("7z Plugin", "7z")
22 ICON "../../Archive/Icons/ntfs.ico" 22 ICON "../../Archive/Icons/ntfs.ico"
23 ICON "../../Archive/Icons/xz.ico" 23 ICON "../../Archive/Icons/xz.ico"
24 ICON "../../Archive/Icons/squashfs.ico" 24 ICON "../../Archive/Icons/squashfs.ico"
25 ICON "../../Archive/Icons/zst.ico"
STRINGTABLE STRINGTABLE
BEGIN BEGIN
100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24" 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25"
END END

21
CPP/7zip/Compress/ZstdDecoder.cpp
View File

@@ -16,8 +16,7 @@ CDecoder::CDecoder():
_buffInSize(ZSTD_DStreamInSize()), _buffInSize(ZSTD_DStreamInSize()),
_buffOutSize(ZSTD_DStreamOutSize()*4), _buffOutSize(ZSTD_DStreamOutSize()*4),
_processedIn(0), _processedIn(0),
_processedOut(0), _processedOut(0)
_propsWereSet(false)
{ {
_props.clear(); _props.clear();
} }
@@ -66,7 +65,6 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
#endif #endif
memcpy(&_props, pProps, sizeof (DProps)); memcpy(&_props, pProps, sizeof (DProps));
_propsWereSet = true;
return S_OK; return S_OK;
} }
@@ -91,9 +89,6 @@ HRESULT CDecoder::CreateDecompressor()
{ {
size_t result; size_t result;
if (!_propsWereSet)
return E_FAIL;
if (!_dstream) { if (!_dstream) {
_dstream = ZSTD_createDStream(); _dstream = ZSTD_createDStream();
if (!_dstream) if (!_dstream)
@@ -198,6 +193,13 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream
/* finished */ /* finished */
if (input.pos == input.size) if (input.pos == input.size)
break; break;
/* end of frame */
if (result == 0) {
result = ZSTD_initDStream(_dstream);
if (ZSTD_isError(result))
return ErrorOut(result);
}
} }
} }
} }
@@ -266,6 +268,13 @@ STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize)
/* finished */ /* finished */
if (input.pos == input.size) if (input.pos == input.size)
break; break;
/* end of frame */
if (result == 0) {
result = ZSTD_initDStream(_dstream);
if (ZSTD_isError(result))
return ErrorOut(result);
}
} }
} }
} }

1
CPP/7zip/Compress/ZstdDecoder.h
View File

@@ -49,7 +49,6 @@ class CDecoder:public ICompressCoder,
CMyComPtr < ISequentialInStream > _inStream; CMyComPtr < ISequentialInStream > _inStream;
DProps _props; DProps _props;
bool _propsWereSet;
ZSTD_DStream *_dstream; ZSTD_DStream *_dstream;
void *_buffIn; void *_buffIn;

2
CPP/7zip/UI/Explorer/ContextMenu.cpp
View File

@@ -804,7 +804,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu,
mi.fMask |= MIIM_CHECKMARKS; mi.fMask |= MIIM_CHECKMARKS;
mi.wID = currentCommandID++; mi.wID = currentCommandID++;
mi.hSubMenu = subMenu; mi.hSubMenu = subMenu;
mi.StringValue.SetFromAscii("CRC SHA ZS"); mi.StringValue.SetFromAscii("CRC SHA ZStandard");
mi.hbmpUnchecked = bitmap; mi.hbmpUnchecked = bitmap;
CMenu menu; CMenu menu;