Merge pull request #10 from Unity-Technologies/master

Fix build since v1.1.4, make build with VS2015, build 7za.exe
This commit is contained in:
Tino Reichardt
2017-04-12 16:55:53 +02:00
committed by GitHub
11 changed files with 174 additions and 35 deletions
+1
View File
@@ -63,6 +63,7 @@ You can contact the author at :
#include "lz4.h"
#include "lz4hc.h"
#define XXH_STATIC_LINKING_ONLY
#define XXH_NAMESPACE lz
#include "xxhash.h"
+1
View File
@@ -105,6 +105,7 @@ static void XXH_free (void* p) { free(p); }
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
#define XXH_STATIC_LINKING_ONLY
#define XXH_NAMESPACE lz
#include "xxhash.h"
+1
View File
@@ -63,6 +63,7 @@ You can contact the author at :
#include "lz5frame_static.h"
#include "lz5.h"
#include "lz5hc.h"
#define XXH_NAMESPACE lz
#include "xxhash.h"
+1
View File
@@ -106,6 +106,7 @@ You can contact the author at :
/**************************************
* Includes & Memory related functions
***************************************/
#define XXH_NAMESPACE lz
#include "xxhash.h"
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
+87 -13
View File
@@ -39,7 +39,7 @@ extern "C" {
zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios
at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and
decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22.
Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory.
Levels >= 20, labeled `--ultra`, should be used with caution, as they require more memory.
Compression can be done in:
- a single step (described as Simple API)
- a single step, reusing a context (described as Explicit memory management)
@@ -56,7 +56,7 @@ extern "C" {
/*------ Version ------*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 3
#define ZSTD_VERSION_RELEASE 4
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str
@@ -80,7 +80,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity,
int compressionLevel);
/*! ZSTD_decompress() :
`compressedSize` : must be the _exact_ size of a single compressed frame.
`compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
`dstCapacity` is an upper bound of originalSize.
If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
@return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
@@ -89,6 +89,15 @@ ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*! ZSTD_getDecompressedSize() :
* NOTE: This function is planned to be obsolete, in favour of ZSTD_getFrameContentSize.
* ZSTD_getFrameContentSize functions the same way, returning the decompressed size of a single
* frame, but distinguishes empty frames from frames with an unknown size, or errors.
*
* Additionally, ZSTD_findDecompressedSize can be used instead. It can handle multiple
* concatenated frames in one buffer, and so is more general.
* As a result however, it requires more computation and entire frames to be passed to it,
* as opposed to ZSTD_getFrameContentSize which requires only a single frame's header.
*
* 'src' is the start of a zstd compressed frame.
* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise.
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
@@ -130,7 +139,11 @@ ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx);
Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */
ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel);
/*= Decompression context */
/*= Decompression context
* When decompressing many times,
* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
* This will make workload friendlier for system's memory.
* Use one context per thread for parallel execution in multi-threaded environments. */
typedef struct ZSTD_DCtx_s ZSTD_DCtx;
ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void);
ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx);
@@ -268,9 +281,11 @@ typedef struct ZSTD_outBuffer_s {
* *******************************************************************/
typedef struct ZSTD_CStream_s ZSTD_CStream;
/*===== ZSTD_CStream management functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void);
ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs);
/*===== Streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output);
@@ -304,9 +319,11 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output
* *******************************************************************************/
typedef struct ZSTD_DStream_s ZSTD_DStream;
/*===== ZSTD_DStream management functions =====*/
ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void);
ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds);
/*===== Streaming decompression functions =====*/
ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds);
ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
@@ -331,9 +348,12 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output
#define ZSTD_MAGICNUMBER 0xFD2FB528 /* >= v0.8.0 */
#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U
#define ZSTD_WINDOWLOG_MAX_32 25
#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1)
#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2)
#define ZSTD_WINDOWLOG_MAX_32 27
#define ZSTD_WINDOWLOG_MAX_64 27
#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
#define ZSTD_WINDOWLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64))
#define ZSTD_WINDOWLOG_MIN 10
#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX
#define ZSTD_HASHLOG_MIN 6
@@ -384,6 +404,54 @@ typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size);
typedef void (*ZSTD_freeFunction) (void* opaque, void* address);
typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem;
/***************************************
* Compressed size functions
***************************************/
/*! ZSTD_findFrameCompressedSize() :
* `src` should point to the start of a ZSTD encoded frame or skippable frame
* `srcSize` must be at least as large as the frame
* @return : the compressed size of the frame pointed to by `src`, suitable to pass to
* `ZSTD_decompress` or similar, or an error code if given invalid input. */
ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize);
/***************************************
* Decompressed size functions
***************************************/
/*! ZSTD_getFrameContentSize() :
* `src` should point to the start of a ZSTD encoded frame
* `srcSize` must be at least as large as the frame header. A value greater than or equal
* to `ZSTD_frameHeaderSize_max` is guaranteed to be large enough in all cases.
* @return : decompressed size of the frame pointed to be `src` if known, otherwise
* - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined
* - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */
ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize);
/*! ZSTD_findDecompressedSize() :
* `src` should point the start of a series of ZSTD encoded and/or skippable frames
* `srcSize` must be the _exact_ size of this series
* (i.e. there should be a frame boundary exactly `srcSize` bytes after `src`)
* @return : the decompressed size of all data in the contained frames, as a 64-bit value _if known_
* - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN
* - if an error occurred: ZSTD_CONTENTSIZE_ERROR
*
* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode.
* When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size.
* In which case, it's necessary to use streaming mode to decompress data.
* Optionally, application can still use ZSTD_decompress() while relying on implied limits.
* (For example, data may be necessarily cut into blocks <= 16 KB).
* note 2 : decompressed size is always present when compression is done with ZSTD_compress()
* note 3 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure result fits within application's authorized limits.
* Each application can set its own limits.
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
* read each contained frame header. This is efficient as most of the data is skipped,
* however it does mean that all frame data must be present and valid. */
ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/***************************************
* Advanced compression functions
@@ -402,7 +470,8 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx);
typedef enum {
ZSTD_p_forceWindow /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0)*/
ZSTD_p_forceWindow, /* Force back-references to remain < windowSize, even when referencing Dictionary content (default:0) */
ZSTD_p_forceRawDict /* Force loading dictionary in "content-only" mode (no header analysis) */
} ZSTD_CCtxParameter;
/*! ZSTD_setCCtxParameter() :
* Set advanced parameters, selected through enum ZSTD_CCtxParameter
@@ -479,6 +548,8 @@ ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx);
* It is important that dictBuffer outlives DDict, it must remain read accessible throughout the lifetime of DDict */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize);
/*! ZSTD_createDDict_advanced() :
* Create a ZSTD_DDict using external alloc and free, optionally by reference */
ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
unsigned byReference, ZSTD_customMem customMem);
@@ -517,12 +588,12 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize);
/*===== Advanced Streaming compression functions =====*/
ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem);
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */
ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /**< note: a dict will not be used if dict == NULL or dictSize < 8 */
ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize,
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */
ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */
ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before. note: pledgedSrcSize must be correct, a size of 0 means unknown. for a frame size of 0 use initCStream_advanced */
ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs);
@@ -578,9 +649,9 @@ ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds);
/*===== Buffer-less streaming compression functions =====*/
ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize);
ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be 0 (meaning unknown). note: if the contentSizeFlag is set, pledgedSrcSize == 0 means the source size is actually 0 */
ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
ZSTDLIB_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize can be 0, indicating unknown size. if it is non-zero, it must be accurate. for 0 size frames, use compressBegin_advanced */
ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize);
@@ -640,6 +711,9 @@ ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapaci
c) Frame Content - any content (User Data) of length equal to Frame Size
For skippable frames ZSTD_decompressContinue() always returns 0.
For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable.
Note : If fparamsPtr->frameContentSize==0, it is ambiguous: the frame might actually be a Zstd encoded frame with no content.
For purposes of decompression, it is valid in both cases to skip the frame using
ZSTD_findFrameCompressedSize to find its size in bytes.
It also returns Frame Size as fparamsPtr->frameContentSize.
*/
+7 -7
View File
@@ -532,18 +532,18 @@ size_t LZ4MT_decompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr)
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->in.buf = 0;
w->in.size = 0;
w->in.allocated = 0;
pthread_create(&w->pthread, NULL, pt_decompress, w);
cwork_t *wt = &ctx->cwork[t];
wt->in.buf = 0;
wt->in.size = 0;
wt->in.allocated = 0;
pthread_create(&wt->pthread, NULL, pt_decompress, wt);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
cwork_t *wt = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
pthread_join(wt->pthread, &p);
if (p)
retval_of_thread = p;
}
+7 -7
View File
@@ -532,18 +532,18 @@ size_t LZ5MT_decompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr)
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->in.buf = 0;
w->in.size = 0;
w->in.allocated = 0;
pthread_create(&w->pthread, NULL, pt_decompress, w);
cwork_t *wt = &ctx->cwork[t];
wt->in.buf = 0;
wt->in.size = 0;
wt->in.allocated = 0;
pthread_create(&wt->pthread, NULL, pt_decompress, wt);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
cwork_t *wt = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
pthread_join(wt->pthread, &p);
if (p)
retval_of_thread = p;
}
+8 -8
View File
@@ -627,10 +627,10 @@ static size_t st_decompress(void *arg)
goto error_clib;
if (zOut.pos) {
ZSTDMT_Buffer w;
w.size = zOut.pos;
w.buf = zOut.dst;
rv = ctx->fn_write(ctx->arg_write, &w);
ZSTDMT_Buffer wb;
wb.size = zOut.pos;
wb.buf = zOut.dst;
rv = ctx->fn_write(ctx->arg_write, &wb);
if (rv != 0) {
result = mt_error(rv);
goto error;
@@ -810,15 +810,15 @@ size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr)
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
pthread_create(&w->pthread, NULL, pt_decompress, w);
cwork_t *wt = &ctx->cwork[t];
pthread_create(&wt->pthread, NULL, pt_decompress, wt);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
cwork_t *wt = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
pthread_join(wt->pthread, &p);
if (p)
retval_of_thread = p;
}
+53
View File
@@ -215,6 +215,59 @@ C_OBJS = \
$O\XzEnc.obj \
$O\XzIn.obj \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\Lz4Decoder.obj \
$O\Lz4Encoder.obj \
$O\Lz4Register.obj \
$O\Lz5Decoder.obj \
$O\Lz5Encoder.obj \
$O\Lz5Register.obj \
$O\ZstdDecoder.obj \
$O\ZstdEncoder.obj \
$O\ZstdRegister.obj \
LZ4_OBJS = \
$O\lz4.obj \
$O\lz4frame.obj \
$O\lz4hc.obj \
$O\lz4xxhash.obj \
LZ5_OBJS = \
$O\lz5.obj \
$O\lz5frame.obj \
$O\lz5hc.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\fse_compress.obj \
$O\huf_compress.obj \
$O\zstd_compress.obj \
$O\error_private.obj \
$O\zstd_v01.obj \
$O\zstd_v02.obj \
$O\zstd_v03.obj \
$O\zstd_v04.obj \
$O\zstd_v05.obj \
$O\zstd_v06.obj \
$O\zstd_v07.obj \
ZSTDMT_OBJS = \
$O\lz5mt_common.obj \
$O\lz5mt_compress.obj \
$O\lz5mt_decompress.obj \
$O\lz4mt_common.obj \
$O\lz4mt_compress.obj \
$O\lz4mt_decompress.obj \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_compress.obj \
$O\zstdmt_decompress.obj \
!include "../../UI/Console/Console.mak"
!include "../../Aes.mak"
+4
View File
@@ -60,5 +60,9 @@ cd %ROOT%\Bundles\SFXCon
nmake %OPTS%
copy O\7zCon.sfx %OUTDIR%\7zCon.sfx
cd %ROOT%\Bundles\Alone
nmake %OPTS%
copy O\7za.exe %OUTDIR%\7za.exe
:ende
cd %ROOT%\..
+4
View File
@@ -60,5 +60,9 @@ cd %ROOT%\Bundles\SFXCon
nmake %OPTS%
copy AMD64\7zCon.sfx %OUTDIR%\7zCon.sfx
cd %ROOT%\Bundles\Alone
nmake %OPTS%
copy AMD64\7za.exe %OUTDIR%\7za.exe
:ende
cd %ROOT%\..