xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-07 11:14:58 -06:00
Code Issues Packages Projects Releases Wiki Activity

merge multi threading to master branch

Browse Source 
- updated zstd to latest devel release
- lz4, lz5 and zstd is included now
- all three support threading
This commit is contained in:
Tino Reichardt
2016-10-16 23:38:46 +02:00
parent f3f39b74b0
commit 58069903d0
108 changed files with 21091 additions and 609 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
C/7zVersion.h
View File

@@ -3,12 +3,12 @@
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "16.04 ZS"
#define MY_VERSION "16.04 ZS"
#define MY_DATE "2016-10-06"
#define MY_DATE "2016-10-16"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"
#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
#define MY_COPYRIGHT_CR "(c) 1999-2016 Igor Pavlov, Tino Reichardt"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2016 Igor Pavlov"
#ifdef USE_COPYRIGHT_CR
#define MY_COPYRIGHT MY_COPYRIGHT_CR

2
C/7zVersionTr.h
View File

@@ -3,7 +3,7 @@
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "1.1.0"
#define MY_VERSION "1.1.0"
#define MY_DATE "2016-10-06"
#define MY_DATE "2016-10-16"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Tino Reichardt"

1516
C/lz4/lz4.c Normal file
View File

File diff suppressed because it is too large Load Diff

360
C/lz4/lz4.h Normal file
View File

@@ -0,0 +1,360 @@
/*
LZ4 - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*
* lz4.h provides block compression functions, and gives full buffer control to programmer.
* If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
* and can let the library handle its own memory, please use lz4frame.h instead.
*/
/**************************************
* Version
**************************************/
#define LZ4_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ4_VERSION_MINOR 7 /* for new (non-breaking) interface capabilities */
#define LZ4_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */
#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
int LZ4_versionNumber (void);
/**************************************
* Tuning parameter
**************************************/
/*
* LZ4_MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
*/
#define LZ4_MEMORY_USAGE 14
/**************************************
* Simple Functions
**************************************/
int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
LZ4_compress_default() :
Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ4_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails
LZ4_decompress_safe() :
compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
*/
/**************************************
* Advanced Functions
**************************************/
#define LZ4_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ4_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
/*
LZ4_compressBound() :
Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
inputSize : max supported value is LZ4_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
*/
int LZ4_compressBound(int inputSize);
/*
LZ4_compress_fast() :
Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ4_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
*/
int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
/*
LZ4_compress_fast_extState() :
Same compression function, just using an externally allocated memory space to store compression state.
Use LZ4_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
*/
int LZ4_sizeofState(void);
int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
/*
LZ4_compress_destSize() :
Reverse the logic, by compressing as much data as possible from 'source' buffer
into already allocated buffer 'dest' of size 'targetDestSize'.
This function either compresses the entire 'source' content into 'dest' if it's large enough,
or fill 'dest' buffer completely with as much data as possible from 'source'.
*sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
New value is necessarily <= old value.
return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
or 0 if compression fails
*/
int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
/*
LZ4_decompress_fast() :
originalSize : is the original and therefore uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size)
If the source stream is detected malformed, the function will stop decoding and return a negative result.
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
note : This function fully respect memory boundaries for properly formed compressed data.
It is a bit faster than LZ4_decompress_safe().
However, it does not provide any protection against intentionally modified data stream (malicious input).
Use this function in trusted environment only (data to decode comes from a trusted source).
*/
int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
/*
LZ4_decompress_safe_partial() :
This function decompress a compressed block of size 'compressedSize' at position 'source'
into destination buffer 'dest' of size 'maxDecompressedSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
*/
int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
/***********************************************
* Streaming Compression Functions
***********************************************/
#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
#define LZ4_STREAMSIZE (LZ4_STREAMSIZE_U64 * sizeof(long long))
/*
* LZ4_stream_t
* information structure to track an LZ4 stream.
* important : init this structure content before first use !
* note : only allocated directly the structure if you are statically linking LZ4
* If you are using liblz4 as a DLL, please use below construction methods instead.
*/
typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
/*
* LZ4_resetStream
* Use this function to init an allocated LZ4_stream_t structure
*/
void LZ4_resetStream (LZ4_stream_t* streamPtr);
/*
* LZ4_createStream will allocate and initialize an LZ4_stream_t structure
* LZ4_freeStream releases its memory.
* In the context of a DLL (liblz4), please use these methods rather than the static struct.
* They are more future proof, in case of a change of LZ4_stream_t size.
*/
LZ4_stream_t* LZ4_createStream(void);
int LZ4_freeStream (LZ4_stream_t* streamPtr);
/*
* LZ4_loadDict
* Use this function to load a static dictionary into LZ4_stream.
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
* Loading a size of 0 is allowed.
* Return : dictionary size, in bytes (necessarily <= 64 KB)
*/
int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
/*
* LZ4_compress_fast_continue
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
* Important : Previous data blocks are assumed to still be present and unmodified !
* 'dst' buffer must be already allocated.
* If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
*/
int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
/*
* LZ4_saveDict
* If previously compressed data block is not guaranteed to remain available at its memory location
* save it into a safer place (char* safeBuffer)
* Note : you don't need to call LZ4_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
*/
int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
/************************************************
* Streaming Decompression Functions
************************************************/
#define LZ4_STREAMDECODESIZE_U64 4
#define LZ4_STREAMDECODESIZE (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t;
/*
* LZ4_streamDecode_t
* information structure to track an LZ4 stream.
* init this structure content using LZ4_setStreamDecode or memset() before first use !
*
* In the context of a DLL (liblz4) please prefer usage of construction methods below.
* They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
* LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
* LZ4_freeStreamDecode releases its memory.
*/
LZ4_streamDecode_t* LZ4_createStreamDecode(void);
int LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
/*
* LZ4_setStreamDecode
* Use this function to instruct where to find the dictionary.
* Setting a size of 0 is allowed (same effect as reset).
* Return : 1 if OK, 0 if error
*/
int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ4_setStreamDecode()
*/
int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
*/
int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
/**************************************
* Obsolete Functions
**************************************/
/* Deprecate Warnings */
/* Should these warnings messages be a problem,
it is generally possible to disable them,
with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual for example.
You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# endif
#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */
/* Obsolete compression functions */
/* These functions are planned to start generate warnings by r131 approximately */
int LZ4_compress (const char* source, char* dest, int sourceSize);
int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
int LZ4_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compress_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
int LZ4_compress_limitedOutput_continue (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete decompression functions */
/* These function names are completely deprecated and must no longer be used.
They are only provided here for compatibility with older programs.
- LZ4_uncompress is the same as LZ4_decompress_fast
- LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
These function prototypes are now disabled; uncomment them only if you really need them.
It is highly recommended to stop using these prototypes and migrate to maintained ones */
/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
/* Obsolete streaming functions; use new streaming interface whenever possible */
LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
LZ4_DEPRECATED("use LZ4_createStream() instead") int LZ4_sizeofStreamState(void);
LZ4_DEPRECATED("use LZ4_resetStream() instead") int LZ4_resetStreamState(void* state, char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDict() instead") char* LZ4_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
#if defined (__cplusplus)
}
#endif

1479
C/lz4/lz4frame.c Normal file
View File

File diff suppressed because it is too large Load Diff

303
C/lz4/lz4frame.h Normal file
View File

@@ -0,0 +1,303 @@
/*
LZ4 auto-framing library
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/* LZ4F is a stand-alone API to create LZ4-compressed frames
* fully conformant to specification v1.5.1.
* All related operations, including memory management, are handled by the library.
* You don't need lz4.h when using lz4frame.h.
* */
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/**************************************
* Includes
**************************************/
#include <stddef.h> /* size_t */
/**************************************
* Error management
* ************************************/
typedef size_t LZ4F_errorCode_t;
unsigned LZ4F_isError(LZ4F_errorCode_t code);
const char* LZ4F_getErrorName(LZ4F_errorCode_t code); /* return error code string; useful for debugging */
/**************************************
* Frame compression types
* ************************************/
//#define LZ4F_DISABLE_OBSOLETE_ENUMS
#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
# define LZ4F_OBSOLETE_ENUM(x) ,x
#else
# define LZ4F_OBSOLETE_ENUM(x)
#endif
typedef enum {
LZ4F_default=0,
LZ4F_max64KB=4,
LZ4F_max256KB=5,
LZ4F_max1MB=6,
LZ4F_max4MB=7
LZ4F_OBSOLETE_ENUM(max64KB = LZ4F_max64KB)
LZ4F_OBSOLETE_ENUM(max256KB = LZ4F_max256KB)
LZ4F_OBSOLETE_ENUM(max1MB = LZ4F_max1MB)
LZ4F_OBSOLETE_ENUM(max4MB = LZ4F_max4MB)
} LZ4F_blockSizeID_t;
typedef enum {
LZ4F_blockLinked=0,
LZ4F_blockIndependent
LZ4F_OBSOLETE_ENUM(blockLinked = LZ4F_blockLinked)
LZ4F_OBSOLETE_ENUM(blockIndependent = LZ4F_blockIndependent)
} LZ4F_blockMode_t;
typedef enum {
LZ4F_noContentChecksum=0,
LZ4F_contentChecksumEnabled
LZ4F_OBSOLETE_ENUM(noContentChecksum = LZ4F_noContentChecksum)
LZ4F_OBSOLETE_ENUM(contentChecksumEnabled = LZ4F_contentChecksumEnabled)
} LZ4F_contentChecksum_t;
typedef enum {
LZ4F_frame=0,
LZ4F_skippableFrame
LZ4F_OBSOLETE_ENUM(skippableFrame = LZ4F_skippableFrame)
} LZ4F_frameType_t;
#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
typedef LZ4F_blockSizeID_t blockSizeID_t;
typedef LZ4F_blockMode_t blockMode_t;
typedef LZ4F_frameType_t frameType_t;
typedef LZ4F_contentChecksum_t contentChecksum_t;
#endif
typedef struct {
LZ4F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ4F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LZ4F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LZ4F_frameType_t frameType; /* LZ4F_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
} LZ4F_frameInfo_t;
typedef struct {
LZ4F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ4F_preferences_t;
/***********************************
* Simple compression function
* *********************************/
size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
/* LZ4F_compressFrame()
* Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
* If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
* The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* The result of the function is the number of bytes written into dstBuffer.
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
*/
/**********************************
* Advanced compression functions
**********************************/
typedef struct LZ4F_cctx_s* LZ4F_compressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */
unsigned reserved[3];
} LZ4F_compressOptions_t;
/* Resource Management */
#define LZ4F_VERSION 100
LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* cctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t cctx);
/* LZ4F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
* The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries.
* The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object.
* If the result LZ4F_errorCode_t is not zero, there was an error during context creation.
* Object can release its memory using LZ4F_freeCompressionContext();
*/
/* Compression */
size_t LZ4F_compressBegin(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
* The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
* The result of the function is the number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
/* LZ4F_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
* Different preferences can produce different results.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
*/
size_t LZ4F_compressUpdate(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressUpdate()
* LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ4F_compressBound().
* If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
* LZ4F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
*/
size_t LZ4F_flush(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_flush()
* Should you need to generate compressed data immediately, without waiting for the current block to be filled,
* you can call LZ4_flush(), which will immediately compress any remaining data buffered within cctx.
* Note that dstMaxSize must be large enough to ensure the operation will be successful.
* LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer
* (it can be zero, this means there was no data left within cctx)
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
*/
size_t LZ4F_compressEnd(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr);
/* LZ4F_compressEnd()
* When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
* It will flush whatever data remained within compressionContext (like LZ4_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* The function outputs an error code if it fails (can be tested using LZ4F_isError())
* The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
* A successful call to LZ4F_compressEnd() makes cctx available again for next compression task.
*/
/***********************************
* Decompression functions
***********************************/
typedef struct LZ4F_dctx_s* LZ4F_decompressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
unsigned reserved[3];
} LZ4F_decompressOptions_t;
/* Resource management */
LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* dctxPtr, unsigned version);
LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t dctx);
/* LZ4F_createDecompressionContext() :
* The first thing to do is to create an LZ4F_decompressionContext_t object, which will be used in all decompression operations.
* This is achieved using LZ4F_createDecompressionContext().
* The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
* The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
* The result is an errorCode, which can be tested using LZ4F_isError().
* dctx memory can be released using LZ4F_freeDecompressionContext();
* The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
/* Decompression */
size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dctx,
LZ4F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/* LZ4F_getFrameInfo()
* This function decodes frame header information (such as max blockSize, frame checksum, etc.).
* Its usage is optional : you can start by calling directly LZ4F_decompress() instead.
* The objective is to extract frame header information, typically for allocation purposes.
* LZ4F_getFrameInfo() can also be used anytime *after* starting decompression, on any valid LZ4F_decompressionContext_t.
* The result is *copied* into an existing LZ4F_frameInfo_t structure which must be already allocated.
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call,
* or an error code which can be tested using LZ4F_isError()
* (typically, when there is not enough src bytes to fully decode the frame header)
* You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
*/
size_t LZ4F_decompress(LZ4F_decompressionContext_t dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ4F_decompressOptions_t* dOptPtr);
/* LZ4F_decompress()
* Call this function repetitively to regenerate data compressed within srcBuffer.
* The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* If number of bytes read is < number of bytes provided, then decompression operation is not completed.
* It typically happens when dstBuffer is not large enough to contain all decoded data.
* LZ4F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr)
* The function will check this condition, and refuse to continue if it is not respected.
*
* dstBuffer is supposed to be flushed between each call to the function, since its content will be overwritten.
* dst arguments can be changed at will with each consecutive call to the function.
*
* The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call.
* Schematically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
* This is just a hint, you can always provide any srcSize you want.
* When a frame is fully decoded, the function result will be 0 (no more data expected).
* If decompression failed, function result is an error code, which can be tested using LZ4F_isError().
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
#if defined (__cplusplus)
}
#endif

81
C/lz4/lz4frame_static.h Normal file
View File

@@ -0,0 +1,81 @@
/*
LZ4 auto-framing library
Header File for static linking only
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* lz4frame_static.h should be used solely in the context of static linking.
* It contains definitions which may still change overtime.
* Never use it in the context of DLL linking.
* */
/**************************************
* Includes
**************************************/
#include "lz4frame.h"
/**************************************
* Error management
* ************************************/
#define LZ4F_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_maxCode)
//#define LZ4F_DISABLE_OLD_ENUMS
#ifndef LZ4F_DISABLE_OLD_ENUMS
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM,
#else
#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
#endif
typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
#if defined (__cplusplus)
}
#endif

731
C/lz4/lz4hc.c Normal file
View File

@@ -0,0 +1,731 @@
/*
LZ4 HC - High Compression Mode of LZ4
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
/**************************************
* Tuning Parameter
**************************************/
static const int LZ4HC_compressionLevel_default = 9;
/**************************************
* Includes
**************************************/
#include "lz4hc.h"
/**************************************
* Local Compiler Options
**************************************/
#if defined(__GNUC__)
# pragma GCC diagnostic ignored "-Wunused-function"
#endif
#if defined (__clang__)
# pragma clang diagnostic ignored "-Wunused-function"
#endif
/**************************************
* Common LZ4 definition
**************************************/
#define LZ4_COMMONDEFS_ONLY
#include "lz4.c"
/**************************************
* Local Constants
**************************************/
#define DICTIONARY_LOGSIZE 16
#define MAXD (1<<DICTIONARY_LOGSIZE)
#define MAXD_MASK (MAXD - 1)
#define HASH_LOG (DICTIONARY_LOGSIZE-1)
#define HASHTABLESIZE (1 << HASH_LOG)
#define HASH_MASK (HASHTABLESIZE - 1)
#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
static const int g_maxCompressionLevel = 16;
/**************************************
* Local Types
**************************************/
typedef struct
{
U32 hashTable[HASHTABLESIZE];
U16 chainTable[MAXD];
const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */
const BYTE* dictBase; /* alternate base for extDict */
BYTE* inputBuffer; /* deprecated */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 compressionLevel;
} LZ4HC_Data_Structure;
/**************************************
* Local Macros
**************************************/
#define HASH_FUNCTION(i) (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
//#define DELTANEXTU16(p) chainTable[(p) & MAXD_MASK] /* flexible, MAXD dependent */
#define DELTANEXTU16(p) chainTable[(U16)(p)] /* faster */
static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
/**************************************
* HC Compression
**************************************/
static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
{
MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
hc4->nextToUpdate = 64 KB;
hc4->base = start - 64 KB;
hc4->end = start;
hc4->dictBase = start - 64 KB;
hc4->dictLimit = 64 KB;
hc4->lowLimit = 64 KB;
}
/* Update chains up to ip (excluded) */
FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
{
U16* chainTable = hc4->chainTable;
U32* HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const U32 target = (U32)(ip - base);
U32 idx = hc4->nextToUpdate;
while(idx < target)
{
U32 h = LZ4HC_hashPtr(base+idx);
size_t delta = idx - HashTable[h];
if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
DELTANEXTU16(idx) = (U16)delta;
HashTable[h] = idx;
idx++;
}
hc4->nextToUpdate = target;
}
FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4, /* Index table will be updated */
const BYTE* ip, const BYTE* const iLimit,
const BYTE** matchpos,
const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const BYTE* const dictBase = hc4->dictBase;
const U32 dictLimit = hc4->dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
U32 matchIndex;
const BYTE* match;
int nbAttempts=maxNbAttempts;
size_t ml=0;
/* HC4 match finder */
LZ4HC_Insert(hc4, ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
while ((matchIndex>=lowLimit) && (nbAttempts))
{
nbAttempts--;
if (matchIndex >= dictLimit)
{
match = base + matchIndex;
if (*(match+ml) == *(ip+ml)
&& (LZ4_read32(match) == LZ4_read32(ip)))
{
size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
if (mlt > ml) { ml = mlt; *matchpos = match; }
}
}
else
{
match = dictBase + matchIndex;
if (LZ4_read32(match) == LZ4_read32(ip))
{
size_t mlt;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iLimit) vLimit = iLimit;
mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iLimit))
mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; } /* virtual matchpos */
}
}
matchIndex -= DELTANEXTU16(matchIndex);
}
return (int)ml;
}
FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
LZ4HC_Data_Structure* hc4,
const BYTE* const ip,
const BYTE* const iLowLimit,
const BYTE* const iHighLimit,
int longest,
const BYTE** matchpos,
const BYTE** startpos,
const int maxNbAttempts)
{
U16* const chainTable = hc4->chainTable;
U32* const HashTable = hc4->hashTable;
const BYTE* const base = hc4->base;
const U32 dictLimit = hc4->dictLimit;
const BYTE* const lowPrefixPtr = base + dictLimit;
const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
const BYTE* const dictBase = hc4->dictBase;
U32 matchIndex;
int nbAttempts = maxNbAttempts;
int delta = (int)(ip-iLowLimit);
/* First Match */
LZ4HC_Insert(hc4, ip);
matchIndex = HashTable[LZ4HC_hashPtr(ip)];
while ((matchIndex>=lowLimit) && (nbAttempts))
{
nbAttempts--;
if (matchIndex >= dictLimit)
{
const BYTE* matchPtr = base + matchIndex;
if (*(iLowLimit + longest) == *(matchPtr - delta + longest))
if (LZ4_read32(matchPtr) == LZ4_read32(ip))
{
int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
int back = 0;
while ((ip+back>iLowLimit)
&& (matchPtr+back > lowPrefixPtr)
&& (ip[back-1] == matchPtr[back-1]))
back--;
mlt -= back;
if (mlt > longest)
{
longest = (int)mlt;
*matchpos = matchPtr+back;
*startpos = ip+back;
}
}
}
else
{
const BYTE* matchPtr = dictBase + matchIndex;
if (LZ4_read32(matchPtr) == LZ4_read32(ip))
{
size_t mlt;
int back=0;
const BYTE* vLimit = ip + (dictLimit - matchIndex);
if (vLimit > iHighLimit) vLimit = iHighLimit;
mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
mlt -= back;
if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
}
}
matchIndex -= DELTANEXTU16(matchIndex);
}
return longest;
}
typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
#define LZ4HC_DEBUG 0
#if LZ4HC_DEBUG
static unsigned debug = 0;
#endif
FORCE_INLINE int LZ4HC_encodeSequence (
const BYTE** ip,
BYTE** op,
const BYTE** anchor,
int matchLength,
const BYTE* const match,
limitedOutput_directive limitedOutputBuffer,
BYTE* oend)
{
int length;
BYTE* token;
#if LZ4HC_DEBUG
if (debug) printf("literal : %u -- match : %u -- offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
#endif
/* Encode Literal length */
length = (int)(*ip - *anchor);
token = (*op)++;
if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1; /* Check output limit */
if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; }
else *token = (BYTE)(length<<ML_BITS);
/* Copy Literals */
LZ4_wildCopy(*op, *anchor, (*op) + length);
*op += length;
/* Encode Offset */
LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
/* Encode MatchLength */
length = (int)(matchLength-MINMATCH);
if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1; /* Check output limit */
if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
else *token += (BYTE)(length);
/* Prepare next loop */
*ip += matchLength;
*anchor = *ip;
return 0;
}
static int LZ4HC_compress_generic (
void* ctxvoid,
const char* source,
char* dest,
int inputSize,
int maxOutputSize,
int compressionLevel,
limitedOutput_directive limit
)
{
LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
const BYTE* ip = (const BYTE*) source;
const BYTE* anchor = ip;
const BYTE* const iend = ip + inputSize;
const BYTE* const mflimit = iend - MFLIMIT;
const BYTE* const matchlimit = (iend - LASTLITERALS);
BYTE* op = (BYTE*) dest;
BYTE* const oend = op + maxOutputSize;
unsigned maxNbAttempts;
int ml, ml2, ml3, ml0;
const BYTE* ref=NULL;
const BYTE* start2=NULL;
const BYTE* ref2=NULL;
const BYTE* start3=NULL;
const BYTE* ref3=NULL;
const BYTE* start0;
const BYTE* ref0;
/* init */
if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default;
maxNbAttempts = 1 << (compressionLevel-1);
ctx->end += inputSize;
ip++;
/* Main Loop */
while (ip < mflimit)
{
ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
if (!ml) { ip++; continue; }
/* saved, in case we would skip too much */
start0 = ip;
ref0 = ref;
ml0 = ml;
_Search2:
if (ip+ml < mflimit)
ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
else ml2 = ml;
if (ml2 == ml) /* No better match */
{
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
continue;
}
if (start0 < ip)
{
if (start2 < ip + ml0) /* empirical */
{
ip = start0;
ref = ref0;
ml = ml0;
}
}
/* Here, start0==ip */
if ((start2 - ip) < 3) /* First Match too small : removed */
{
ml = ml2;
ip = start2;
ref =ref2;
goto _Search2;
}
_Search3:
/*
* Currently we have :
* ml2 > ml1, and
* ip1+3 <= ip2 (usually < ip1+ml1)
*/
if ((start2 - ip) < OPTIMAL_ML)
{
int correction;
int new_ml = ml;
if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
correction = new_ml - (int)(start2 - ip);
if (correction > 0)
{
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
}
/* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
if (start2 + ml2 < mflimit)
ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
else ml3 = ml2;
if (ml3 == ml2) /* No better match : 2 sequences to encode */
{
/* ip & ref are known; Now for ml */
if (start2 < ip+ml) ml = (int)(start2 - ip);
/* Now, encode 2 sequences */
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
ip = start2;
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
continue;
}
if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */
{
if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
{
if (start2 < ip+ml)
{
int correction = (int)(ip+ml - start2);
start2 += correction;
ref2 += correction;
ml2 -= correction;
if (ml2 < MINMATCH)
{
start2 = start3;
ref2 = ref3;
ml2 = ml3;
}
}
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
ip = start3;
ref = ref3;
ml = ml3;
start0 = start2;
ref0 = ref2;
ml0 = ml2;
goto _Search2;
}
start2 = start3;
ref2 = ref3;
ml2 = ml3;
goto _Search3;
}
/*
* OK, now we have 3 ascending matches; let's write at least the first one
* ip & ref are known; Now for ml
*/
if (start2 < ip+ml)
{
if ((start2 - ip) < (int)ML_MASK)
{
int correction;
if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
correction = ml - (int)(start2 - ip);
if (correction > 0)
{
start2 += correction;
ref2 += correction;
ml2 -= correction;
}
}
else
{
ml = (int)(start2 - ip);
}
}
if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
ip = start2;
ref = ref2;
ml = ml2;
start2 = start3;
ref2 = ref3;
ml2 = ml3;
goto _Search3;
}
/* Encode Last Literals */
{
int lastRun = (int)(iend - anchor);
if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0; /* Check output limit */
if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
else *op++ = (BYTE)(lastRun<<ML_BITS);
memcpy(op, anchor, iend - anchor);
op += iend-anchor;
}
/* End */
return (int) (((char*)op)-dest);
}
int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); }
int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0; /* Error : state is not aligned for pointers (32 or 64 bits) */
LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src);
if (maxDstSize < LZ4_compressBound(srcSize))
return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
else
return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
}
int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
{
LZ4HC_Data_Structure state;
return LZ4_compress_HC_extStateHC(&state, src, dst, srcSize, maxDstSize, compressionLevel);
}
/**************************************
* Streaming Functions
**************************************/
/* allocation */
LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
int LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
/* initialization */
void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
{
LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t)); /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL;
((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel;
}
int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
{
LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr;
if (dictSize > 64 KB)
{
dictionary += dictSize - 64 KB;
dictSize = 64 KB;
}
LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
ctxPtr->end = (const BYTE*)dictionary + dictSize;
return dictSize;
}
/* compression */
static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
{
if (ctxPtr->end >= ctxPtr->base + 4)
LZ4HC_Insert (ctxPtr, ctxPtr->end-3); /* Referencing remaining dictionary content */
/* Only one memory segment for extDict, so any previous extDict is lost at this stage */
ctxPtr->lowLimit = ctxPtr->dictLimit;
ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
ctxPtr->dictBase = ctxPtr->base;
ctxPtr->base = newBlock - ctxPtr->dictLimit;
ctxPtr->end = newBlock;
ctxPtr->nextToUpdate = ctxPtr->dictLimit; /* match referencing will resume from there */
}
static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
const char* source, char* dest,
int inputSize, int maxOutputSize, limitedOutput_directive limit)
{
/* auto-init if forgotten */
if (ctxPtr->base == NULL)
LZ4HC_init (ctxPtr, (const BYTE*) source);
/* Check overflow */
if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
{
size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
if (dictSize > 64 KB) dictSize = 64 KB;
LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
}
/* Check if blocks follow each other */
if ((const BYTE*)source != ctxPtr->end)
LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
/* Check overlapping input/dictionary space */
{
const BYTE* sourceEnd = (const BYTE*) source + inputSize;
const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
const BYTE* dictEnd = ctxPtr->dictBase + ctxPtr->dictLimit;
if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd))
{
if (sourceEnd > dictEnd) sourceEnd = dictEnd;
ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
}
}
return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
}
int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
{
if (maxOutputSize < LZ4_compressBound(inputSize))
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
else
return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
}
/* dictionary saving */
int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
{
LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr;
int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
if (dictSize > 64 KB) dictSize = 64 KB;
if (dictSize < 4) dictSize = 0;
if (dictSize > prefixSize) dictSize = prefixSize;
memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
{
U32 endIndex = (U32)(streamPtr->end - streamPtr->base);
streamPtr->end = (const BYTE*)safeBuffer + dictSize;
streamPtr->base = streamPtr->end - endIndex;
streamPtr->dictLimit = endIndex - dictSize;
streamPtr->lowLimit = endIndex - dictSize;
if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
}
return dictSize;
}
/***********************************
* Deprecated Functions
***********************************/
/* Deprecated compression functions */
/* These functions are planned to start generate warnings by r131 approximately */
int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); }
int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); }
int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); }
int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); }
int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); }
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); }
/* Deprecated streaming functions */
/* These functions currently generate deprecation warnings */
int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
{
if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1; /* Error : pointer is not aligned for pointer (32 or 64 bits) */
LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer;
return 0;
}
void* LZ4_createHC (char* inputBuffer)
{
void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure));
if (hc4 == NULL) return NULL; /* not enough memory */
LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer;
return hc4;
}
int LZ4_freeHC (void* LZ4HC_Data)
{
FREEMEM(LZ4HC_Data);
return (0);
}
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
}
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
{
return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
}
char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
{
LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
return (char*)(hc4->inputBuffer + dictSize);
}

189
C/lz4/lz4hc.h Normal file
View File

@@ -0,0 +1,189 @@
/*
LZ4 HC - High Compression Mode of LZ4
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ4 source repository : https://github.com/Cyan4973/lz4
- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*****************************
* Includes
*****************************/
#include <stddef.h> /* size_t */
/**************************************
* Block Compression
**************************************/
int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
LZ4_compress_HC :
Destination buffer 'dst' must be already allocated.
Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible)
Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
srcSize : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work.
0 means "use default value" (see lz4hc.c).
Values >16 behave the same as 16.
return : the number of bytes written into buffer 'dst'
or 0 if compression fails.
*/
/* Note :
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
*/
int LZ4_sizeofStateHC(void);
int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
LZ4_compress_HC_extStateHC() :
Use this function if you prefer to manually allocate memory for compression tables.
To know how much memory must be allocated for the compression tables, use :
int LZ4_sizeofStateHC();
Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
The allocated memory can then be provided to the compression functions using 'void* state' parameter.
LZ4_compress_HC_extStateHC() is equivalent to previously described function.
It just uses externally allocated memory for stateHC.
*/
/**************************************
* Streaming Compression
**************************************/
#define LZ4_STREAMHCSIZE 262192
#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t;
/*
LZ4_streamHC_t
This structure allows static allocation of LZ4 HC streaming state.
State must then be initialized using LZ4_resetStreamHC() before first use.
Static allocation should only be used in combination with static linking.
If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof.
*/
LZ4_streamHC_t* LZ4_createStreamHC(void);
int LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
/*
These functions create and release memory for LZ4 HC streaming state.
Newly created states are already initialized.
Existing state space can be re-used anytime using LZ4_resetStreamHC().
If you use LZ4 as a DLL, use these functions instead of static structure allocation,
to avoid size mismatch between different versions.
*/
void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
int LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
/*
These functions compress data in successive blocks of any size, using previous blocks as dictionary.
One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
There is an exception for ring buffers, which can be smaller 64 KB.
Such case is automatically detected and correctly handled by LZ4_compress_HC_continue().
Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
Then, use LZ4_compress_HC_continue() to compress each successive block.
It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation.
If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
you must save it to a safer memory space, using LZ4_saveDictHC().
Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
*/
/**************************************
* Deprecated Functions
**************************************/
/* Deprecate Warnings */
/* Should these warnings messages be a problem,
it is generally possible to disable them,
with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual for example.
You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_DEPRECATE_WARNING_DEFBLOCK
# define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
# if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
# define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ4_GCC_VERSION >= 301)
# define LZ4_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ4_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
# define LZ4_DEPRECATED(message)
# endif
#endif // LZ4_DEPRECATE_WARNING_DEFBLOCK
/* compression functions */
/* these functions are planned to trigger warning messages by r131 approximately */
int LZ4_compressHC (const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
int LZ4_compressHC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Streaming functions following the older model; should no longer be used */
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
LZ4_DEPRECATED("use LZ4_saveDictHC() instead") char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_freeStreamHC() instead") int LZ4_freeHC (void* LZ4HC_Data);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int LZ4_sizeofStreamStateHC(void);
LZ4_DEPRECATED("use LZ4_resetStreamHC() instead") int LZ4_resetStreamStateHC(void* state, char* inputBuffer);
#if defined (__cplusplus)
}
#endif

915
C/lz4/lz4xxhash.c Normal file
View File

@@ -0,0 +1,915 @@
/*
xxHash - Fast Hash algorithm
Copyright (C) 2012-2015, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/**************************************
* Tuning parameters
**************************************/
/* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
* You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
*/
#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
# define XXH_USE_UNALIGNED_ACCESS 1
#endif
/* XXH_ACCEPT_NULL_INPUT_POINTER :
* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
* By default, this option is disabled. To enable it, uncomment below define :
*/
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
/* XXH_FORCE_NATIVE_FORMAT :
* By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
* Results are therefore identical for little-endian and big-endian CPU.
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
* Should endian-independance be of no importance for your application, you may set the #define below to 1.
* It will improve speed for Big-endian CPU.
* This option has no impact on Little_Endian CPU.
*/
#define XXH_FORCE_NATIVE_FORMAT 0
/**************************************
* Compiler Specific Options
***************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# define FORCE_INLINE static __forceinline
#else
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/**************************************
* Includes & Memory related functions
***************************************/
#include "xxhash.h"
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
#include <stdlib.h>
static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
/* for memcpy() */
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
/**************************************
* Basic Types
***************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
static U32 XXH_read32(const void* memPtr)
{
U32 val32;
memcpy(&val32, memPtr, 4);
return val32;
}
static U64 XXH_read64(const void* memPtr)
{
U64 val64;
memcpy(&val64, memPtr, 8);
return val64;
}
/******************************************
* Compiler-specific Functions and Macros
******************************************/
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
#if defined(_MSC_VER)
# define XXH_rotl32(x,r) _rotl(x,r)
# define XXH_rotl64(x,r) _rotl64(x,r)
#else
# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
#endif
#if defined(_MSC_VER) /* Visual Studio */
# define XXH_swap32 _byteswap_ulong
# define XXH_swap64 _byteswap_uint64
#elif GCC_VERSION >= 403
# define XXH_swap32 __builtin_bswap32
# define XXH_swap64 __builtin_bswap64
#else
static U32 XXH_swap32 (U32 x)
{
return ((x << 24) & 0xff000000 ) |
((x << 8) & 0x00ff0000 ) |
((x >> 8) & 0x0000ff00 ) |
((x >> 24) & 0x000000ff );
}
static U64 XXH_swap64 (U64 x)
{
return ((x << 56) & 0xff00000000000000ULL) |
((x << 40) & 0x00ff000000000000ULL) |
((x << 24) & 0x0000ff0000000000ULL) |
((x << 8) & 0x000000ff00000000ULL) |
((x >> 8) & 0x00000000ff000000ULL) |
((x >> 24) & 0x0000000000ff0000ULL) |
((x >> 40) & 0x000000000000ff00ULL) |
((x >> 56) & 0x00000000000000ffULL);
}
#endif
/***************************************
* Architecture Macros
***************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */
static const int one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one))
#endif
/*****************************
* Memory reads
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
else
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
else
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
}
FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
/***************************************
* Macros
***************************************/
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
/***************************************
* Constants
***************************************/
#define PRIME32_1 2654435761U
#define PRIME32_2 2246822519U
#define PRIME32_3 3266489917U
#define PRIME32_4 668265263U
#define PRIME32_5 374761393U
#define PRIME64_1 11400714785074694791ULL
#define PRIME64_2 14029467366897019727ULL
#define PRIME64_3 1609587929392839161ULL
#define PRIME64_4 9650029242287828579ULL
#define PRIME64_5 2870177450012600261ULL
/*****************************
* Simple Hash Functions
*****************************/
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
U32 h32;
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL)
{
len=0;
bEnd=p=(const BYTE*)(size_t)16;
}
#endif
if (len>=16)
{
const BYTE* const limit = bEnd - 16;
U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0;
U32 v4 = seed - PRIME32_1;
do
{
v1 += XXH_get32bits(p) * PRIME32_2;
v1 = XXH_rotl32(v1, 13);
v1 *= PRIME32_1;
p+=4;
v2 += XXH_get32bits(p) * PRIME32_2;
v2 = XXH_rotl32(v2, 13);
v2 *= PRIME32_1;
p+=4;
v3 += XXH_get32bits(p) * PRIME32_2;
v3 = XXH_rotl32(v3, 13);
v3 *= PRIME32_1;
p+=4;
v4 += XXH_get32bits(p) * PRIME32_2;
v4 = XXH_rotl32(v4, 13);
v4 *= PRIME32_1;
p+=4;
}
while (p<=limit);
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
}
else
{
h32 = seed + PRIME32_5;
}
h32 += (U32) len;
while (p+4<=bEnd)
{
h32 += XXH_get32bits(p) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
p+=4;
}
while (p<bEnd)
{
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
p++;
}
h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
h32 *= PRIME32_3;
h32 ^= h32 >> 16;
return h32;
}
unsigned XXH32 (const void* input, size_t len, unsigned seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH32_state_t state;
XXH32_reset(&state, seed);
XXH32_update(&state, input, len);
return XXH32_digest(&state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
}
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
U64 h64;
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL)
{
len=0;
bEnd=p=(const BYTE*)(size_t)32;
}
#endif
if (len>=32)
{
const BYTE* const limit = bEnd - 32;
U64 v1 = seed + PRIME64_1 + PRIME64_2;
U64 v2 = seed + PRIME64_2;
U64 v3 = seed + 0;
U64 v4 = seed - PRIME64_1;
do
{
v1 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
v2 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
v3 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
v4 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
}
while (p<=limit);
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
v1 *= PRIME64_2;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
h64 ^= v1;
h64 = h64 * PRIME64_1 + PRIME64_4;
v2 *= PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
h64 ^= v2;
h64 = h64 * PRIME64_1 + PRIME64_4;
v3 *= PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
h64 ^= v3;
h64 = h64 * PRIME64_1 + PRIME64_4;
v4 *= PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
h64 ^= v4;
h64 = h64 * PRIME64_1 + PRIME64_4;
}
else
{
h64 = seed + PRIME64_5;
}
h64 += (U64) len;
while (p+8<=bEnd)
{
U64 k1 = XXH_get64bits(p);
k1 *= PRIME64_2;
k1 = XXH_rotl64(k1,31);
k1 *= PRIME64_1;
h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
if (p+4<=bEnd)
{
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
while (p<bEnd)
{
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH64_state_t state;
XXH64_reset(&state, seed);
XXH64_update(&state, input, len);
return XXH64_digest(&state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
}
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
/****************************************************
* Advanced Hash Functions
****************************************************/
/*** Allocation ***/
typedef struct
{
U64 total_len;
U32 seed;
U32 v1;
U32 v2;
U32 v3;
U32 v4;
U32 mem32[4]; /* defined as U32 for alignment */
U32 memsize;
} XXH_istate32_t;
typedef struct
{
U64 total_len;
U64 seed;
U64 v1;
U64 v2;
U64 v3;
U64 v4;
U64 mem64[4]; /* defined as U64 for alignment */
U32 memsize;
} XXH_istate64_t;
XXH32_state_t* XXH32_createState(void)
{
XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
XXH64_state_t* XXH64_createState(void)
{
XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
/*** Hash feed ***/
XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed)
{
XXH_istate32_t* state = (XXH_istate32_t*) state_in;
state->seed = seed;
state->v1 = seed + PRIME32_1 + PRIME32_2;
state->v2 = seed + PRIME32_2;
state->v3 = seed + 0;
state->v4 = seed - PRIME32_1;
state->total_len = 0;
state->memsize = 0;
return XXH_OK;
}
XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed)
{
XXH_istate64_t* state = (XXH_istate64_t*) state_in;
state->seed = seed;
state->v1 = seed + PRIME64_1 + PRIME64_2;
state->v2 = seed + PRIME64_2;
state->v3 = seed + 0;
state->v4 = seed - PRIME64_1;
state->total_len = 0;
state->memsize = 0;
return XXH_OK;
}
FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
{
XXH_istate32_t* state = (XXH_istate32_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
if (state->memsize + len < 16) /* fill in tmp buffer */
{
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
if (state->memsize) /* some data left from previous update */
{
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
{
const U32* p32 = state->mem32;
state->v1 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v1 = XXH_rotl32(state->v1, 13);
state->v1 *= PRIME32_1;
p32++;
state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v2 = XXH_rotl32(state->v2, 13);
state->v2 *= PRIME32_1;
p32++;
state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v3 = XXH_rotl32(state->v3, 13);
state->v3 *= PRIME32_1;
p32++;
state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v4 = XXH_rotl32(state->v4, 13);
state->v4 *= PRIME32_1;
p32++;
}
p += 16-state->memsize;
state->memsize = 0;
}
if (p <= bEnd-16)
{
const BYTE* const limit = bEnd - 16;
U32 v1 = state->v1;
U32 v2 = state->v2;
U32 v3 = state->v3;
U32 v4 = state->v4;
do
{
v1 += XXH_readLE32(p, endian) * PRIME32_2;
v1 = XXH_rotl32(v1, 13);
v1 *= PRIME32_1;
p+=4;
v2 += XXH_readLE32(p, endian) * PRIME32_2;
v2 = XXH_rotl32(v2, 13);
v2 *= PRIME32_1;
p+=4;
v3 += XXH_readLE32(p, endian) * PRIME32_2;
v3 = XXH_rotl32(v3, 13);
v3 *= PRIME32_1;
p+=4;
v4 += XXH_readLE32(p, endian) * PRIME32_2;
v4 = XXH_rotl32(v4, 13);
v4 *= PRIME32_1;
p+=4;
}
while (p<=limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd)
{
XXH_memcpy(state->mem32, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
return XXH_OK;
}
XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
else
return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
{
const XXH_istate32_t* state = (const XXH_istate32_t*) state_in;
const BYTE * p = (const BYTE*)state->mem32;
const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
if (state->total_len >= 16)
{
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
}
else
{
h32 = state->seed + PRIME32_5;
}
h32 += (U32) state->total_len;
while (p+4<=bEnd)
{
h32 += XXH_readLE32(p, endian) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
p+=4;
}
while (p<bEnd)
{
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1;
p++;
}
h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
h32 *= PRIME32_3;
h32 ^= h32 >> 16;
return h32;
}
U32 XXH32_digest (const XXH32_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_digest_endian(state_in, XXH_littleEndian);
else
return XXH32_digest_endian(state_in, XXH_bigEndian);
}
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
{
XXH_istate64_t * state = (XXH_istate64_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
if (state->memsize + len < 32) /* fill in tmp buffer */
{
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
if (state->memsize) /* some data left from previous update */
{
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
{
const U64* p64 = state->mem64;
state->v1 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v1 = XXH_rotl64(state->v1, 31);
state->v1 *= PRIME64_1;
p64++;
state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v2 = XXH_rotl64(state->v2, 31);
state->v2 *= PRIME64_1;
p64++;
state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v3 = XXH_rotl64(state->v3, 31);
state->v3 *= PRIME64_1;
p64++;
state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v4 = XXH_rotl64(state->v4, 31);
state->v4 *= PRIME64_1;
p64++;
}
p += 32-state->memsize;
state->memsize = 0;
}
if (p+32 <= bEnd)
{
const BYTE* const limit = bEnd - 32;
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
do
{
v1 += XXH_readLE64(p, endian) * PRIME64_2;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
p+=8;
v2 += XXH_readLE64(p, endian) * PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
p+=8;
v3 += XXH_readLE64(p, endian) * PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
p+=8;
v4 += XXH_readLE64(p, endian) * PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
p+=8;
}
while (p<=limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd)
{
XXH_memcpy(state->mem64, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
return XXH_OK;
}
XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
else
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
{
const XXH_istate64_t * state = (const XXH_istate64_t *) state_in;
const BYTE * p = (const BYTE*)state->mem64;
const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
if (state->total_len >= 32)
{
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
v1 *= PRIME64_2;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
h64 ^= v1;
h64 = h64*PRIME64_1 + PRIME64_4;
v2 *= PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
h64 ^= v2;
h64 = h64*PRIME64_1 + PRIME64_4;
v3 *= PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
h64 ^= v3;
h64 = h64*PRIME64_1 + PRIME64_4;
v4 *= PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
h64 ^= v4;
h64 = h64*PRIME64_1 + PRIME64_4;
}
else
{
h64 = state->seed + PRIME64_5;
}
h64 += (U64) state->total_len;
while (p+8<=bEnd)
{
U64 k1 = XXH_readLE64(p, endian);
k1 *= PRIME64_2;
k1 = XXH_rotl64(k1,31);
k1 *= PRIME64_1;
h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
if (p+4<=bEnd)
{
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
while (p<bEnd)
{
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
unsigned long long XXH64_digest (const XXH64_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_digest_endian(state_in, XXH_littleEndian);
else
return XXH64_digest_endian(state_in, XXH_bigEndian);
}

192
C/lz4/xxhash.h Normal file
View File

@@ -0,0 +1,192 @@
/*
xxHash - Extremely Fast Hash algorithm
Header File
Copyright (C) 2012-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* Notice extracted from xxHash homepage :
xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
It also successfully passes all tests from the SMHasher suite.
Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
Name Speed Q.Score Author
xxHash 5.4 GB/s 10
CrapWow 3.2 GB/s 2 Andrew
MumurHash 3a 2.7 GB/s 10 Austin Appleby
SpookyHash 2.0 GB/s 10 Bob Jenkins
SBox 1.4 GB/s 9 Bret Mulvey
Lookup3 1.2 GB/s 9 Bob Jenkins
SuperFastHash 1.2 GB/s 1 Paul Hsieh
CityHash64 1.05 GB/s 10 Pike & Alakuijala
FNV 0.55 GB/s 5 Fowler, Noll, Vo
CRC32 0.43 GB/s 9
MD5-32 0.33 GB/s 10 Ronald L. Rivest
SHA1-32 0.28 GB/s 10
Q.Score is a measure of quality of the hash function.
It depends on successfully passing SMHasher test set.
10 is a perfect score.
A 64-bits version, named XXH64, is available since r35.
It offers much better speed, but for 64-bits applications only.
Name Speed on 64 bits Speed on 32 bits
XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*****************************
* Definitions
*****************************/
#include <stddef.h> /* size_t */
typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
/*****************************
* Namespace Emulation
*****************************/
/* Motivations :
If you need to include xxHash into your library,
but wish to avoid xxHash symbols to be present on your library interface
in an effort to avoid potential name collision if another library also includes xxHash,
you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash
with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values).
Note that no change is required within the calling program :
it can still call xxHash functions using their regular name.
They will be automatically translated by this header.
*/
#ifdef XXH_NAMESPACE
# define XXH_CAT(A,B) A##B
# define XXH_NAME2(A,B) XXH_CAT(A,B)
# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
#endif
/*****************************
* Simple Hash Functions
*****************************/
unsigned int XXH32 (const void* input, size_t length, unsigned seed);
unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
/*
XXH32() :
Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
The memory between input & input+length must be valid (allocated and read-accessible).
"seed" can be used to alter the result predictably.
This function successfully passes all SMHasher tests.
Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
XXH64() :
Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
Faster on 64-bits systems. Slower on 32-bits systems.
*/
/*****************************
* Advanced Hash Functions
*****************************/
typedef struct { long long ll[ 6]; } XXH32_state_t;
typedef struct { long long ll[11]; } XXH64_state_t;
/*
These structures allow static allocation of XXH states.
States must then be initialized using XXHnn_reset() before first use.
If you prefer dynamic allocation, please refer to functions below.
*/
XXH32_state_t* XXH32_createState(void);
XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
XXH64_state_t* XXH64_createState(void);
XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
/*
These functions create and release memory for XXH state.
States must then be initialized using XXHnn_reset() before first use.
*/
XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed);
XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
unsigned int XXH32_digest (const XXH32_state_t* statePtr);
XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
/*
These functions calculate the xxHash of an input provided in multiple smaller packets,
as opposed to an input provided as a single block.
XXH state space must first be allocated, using either static or dynamic method provided above.
Start a new hash by initializing state with a seed, using XXHnn_reset().
Then, feed the hash state by calling XXHnn_update() as many times as necessary.
Obviously, input must be valid, meaning allocated and read accessible.
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, you can produce a hash anytime, by using XXHnn_digest().
This function returns the final nn-bits hash.
You can nonetheless continue feeding the hash state with more input,
and therefore get some new hashes, by calling again XXHnn_digest().
When you are done, don't forget to free XXH state space, using typically XXHnn_freeState().
*/
#if defined (__cplusplus)
}
#endif

1355
C/lz5/lz5.c Normal file
View File

File diff suppressed because it is too large Load Diff

363
C/lz5/lz5.h Normal file
View File

@@ -0,0 +1,363 @@
/*
LZ5 - Fast LZ compression algorithm
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
- LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*
* lz5.h provides block compression functions, and gives full buffer control to programmer.
* If you need to generate inter-operable compressed data (respecting LZ5 frame specification),
* and can let the library handle its own memory, please use lz5frame.h instead.
*/
/**************************************
* Version
**************************************/
#define LZ5_VERSION "v1.5.0"
#define LZ5_VERSION_MAJOR 1 /* for breaking interface changes */
#define LZ5_VERSION_MINOR 5 /* for new (non-breaking) interface capabilities */
#define LZ5_VERSION_RELEASE 0 /* for tweaks, bug-fixes, or development */
#define LZ5_VERSION_NUMBER (LZ5_VERSION_MAJOR *100*100 + LZ5_VERSION_MINOR *100 + LZ5_VERSION_RELEASE)
int LZ5_versionNumber (void);
#define LZ5HC_MAX_CLEVEL 15
/**************************************
* Tuning parameter
**************************************/
/*
* LZ5_MEMORY_USAGE :
* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
*/
#define LZ5_MEMORY_USAGE 20
/**************************************
* Simple Functions
**************************************/
int LZ5_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
int LZ5_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
/*
LZ5_compress_default() :
Compresses 'sourceSize' bytes from buffer 'source'
into already allocated 'dest' buffer of size 'maxDestSize'.
Compression is guaranteed to succeed if 'maxDestSize' >= LZ5_compressBound(sourceSize).
It also runs faster, so it's a recommended setting.
If the function cannot compress 'source' into a more limited 'dest' budget,
compression stops *immediately*, and the function result is zero.
As a consequence, 'dest' content is not valid.
This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
sourceSize : Max supported value is LZ5_MAX_INPUT_VALUE
maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
or 0 if compression fails
LZ5_decompress_safe() :
compressedSize : is the precise full size of the compressed block.
maxDecompressedSize : is the size of destination buffer, which must be already allocated.
return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
If destination buffer is not large enough, decoding will stop and output an error code (<0).
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function is protected against buffer overflow exploits, including malicious data packets.
It never writes outside output buffer, nor reads outside input buffer.
*/
/**************************************
* Advanced Functions
**************************************/
#define LZ5_MAX_INPUT_SIZE 0x7E000000 /* 2 113 929 216 bytes */
#define LZ5_COMPRESSBOUND(isize) ((unsigned)(isize) > (unsigned)LZ5_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/128) + 16)
/*
LZ5_compressBound() :
Provides the maximum size that LZ5 compression may output in a "worst case" scenario (input data not compressible)
This function is primarily useful for memory allocation purposes (destination buffer size).
Macro LZ5_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
Note that LZ5_compress_default() compress faster when dest buffer size is >= LZ5_compressBound(srcSize)
inputSize : max supported value is LZ5_MAX_INPUT_SIZE
return : maximum output size in a "worst case" scenario
or 0, if input size is too large ( > LZ5_MAX_INPUT_SIZE)
*/
int LZ5_compressBound(int inputSize);
/*
LZ5_compress_fast() :
Same as LZ5_compress_default(), but allows to select an "acceleration" factor.
The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
An acceleration value of "1" is the same as regular LZ5_compress_default()
Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz5.c), which is 1.
*/
int LZ5_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
/*
LZ5_compress_fast_extState() :
Same compression function, just using an externally allocated memory space to store compression state.
Use LZ5_sizeofState() to know how much memory must be allocated,
and allocate it on 8-bytes boundaries (using malloc() typically).
Then, provide it as 'void* state' to compression function.
*/
int LZ5_sizeofState(void);
int LZ5_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
/*
LZ5_compress_destSize() :
Reverse the logic, by compressing as much data as possible from 'source' buffer
into already allocated buffer 'dest' of size 'targetDestSize'.
This function either compresses the entire 'source' content into 'dest' if it's large enough,
or fill 'dest' buffer completely with as much data as possible from 'source'.
*sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
New value is necessarily <= old value.
return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
or 0 if compression fails
*/
int LZ5_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
/*
LZ5_decompress_fast() :
originalSize : is the original and therefore uncompressed size
return : the number of bytes read from the source buffer (in other words, the compressed size)
If the source stream is detected malformed, the function will stop decoding and return a negative result.
Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
note : This function fully respect memory boundaries for properly formed compressed data.
It is a bit faster than LZ5_decompress_safe().
However, it does not provide any protection against intentionally modified data stream (malicious input).
Use this function in trusted environment only (data to decode comes from a trusted source).
*/
int LZ5_decompress_fast (const char* source, char* dest, int originalSize);
/*
LZ5_decompress_safe_partial() :
This function decompress a compressed block of size 'compressedSize' at position 'source'
into destination buffer 'dest' of size 'maxDecompressedSize'.
The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
reducing decompression time.
return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
Always control how many bytes were decoded.
If the source stream is detected malformed, the function will stop decoding and return a negative result.
This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
*/
int LZ5_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
/***********************************************
* Streaming Compression Functions
***********************************************/
#define LZ5_STREAMSIZE_U64 ((1 << (LZ5_MEMORY_USAGE-3)) + 4)
#define LZ5_STREAMSIZE (LZ5_STREAMSIZE_U64 * sizeof(long long))
/*
* LZ5_stream_t
* information structure to track an LZ5 stream.
* important : init this structure content before first use !
* note : only allocated directly the structure if you are statically linking LZ5
* If you are using liblz5 as a DLL, please use below construction methods instead.
*/
typedef struct { long long table[LZ5_STREAMSIZE_U64]; } LZ5_stream_t;
/*
* LZ5_resetStream
* Use this function to init an allocated LZ5_stream_t structure
*/
void LZ5_resetStream (LZ5_stream_t* streamPtr);
/*
* LZ5_createStream will allocate and initialize an LZ5_stream_t structure
* LZ5_freeStream releases its memory.
* In the context of a DLL (liblz5), please use these methods rather than the static struct.
* They are more future proof, in case of a change of LZ5_stream_t size.
*/
LZ5_stream_t* LZ5_createStream(void);
int LZ5_freeStream (LZ5_stream_t* streamPtr);
/*
* LZ5_loadDict
* Use this function to load a static dictionary into LZ5_stream.
* Any previous data will be forgotten, only 'dictionary' will remain in memory.
* Loading a size of 0 is allowed.
* Return : dictionary size, in bytes (necessarily <= 64 KB)
*/
int LZ5_loadDict (LZ5_stream_t* streamPtr, const char* dictionary, int dictSize);
/*
* LZ5_compress_fast_continue
* Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
* Important : Previous data blocks are assumed to still be present and unmodified !
* 'dst' buffer must be already allocated.
* If maxDstSize >= LZ5_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
* If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
*/
int LZ5_compress_fast_continue (LZ5_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
/*
* LZ5_saveDict
* If previously compressed data block is not guaranteed to remain available at its memory location
* save it into a safer place (char* safeBuffer)
* Note : you don't need to call LZ5_loadDict() afterwards,
* dictionary is immediately usable, you can therefore call LZ5_compress_fast_continue()
* Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
*/
int LZ5_saveDict (LZ5_stream_t* streamPtr, char* safeBuffer, int dictSize);
/************************************************
* Streaming Decompression Functions
************************************************/
#define LZ5_STREAMDECODESIZE_U64 4
#define LZ5_STREAMDECODESIZE (LZ5_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
typedef struct { unsigned long long table[LZ5_STREAMDECODESIZE_U64]; } LZ5_streamDecode_t;
/*
* LZ5_streamDecode_t
* information structure to track an LZ5 stream.
* init this structure content using LZ5_setStreamDecode or memset() before first use !
*
* In the context of a DLL (liblz5) please prefer usage of construction methods below.
* They are more future proof, in case of a change of LZ5_streamDecode_t size in the future.
* LZ5_createStreamDecode will allocate and initialize an LZ5_streamDecode_t structure
* LZ5_freeStreamDecode releases its memory.
*/
LZ5_streamDecode_t* LZ5_createStreamDecode(void);
int LZ5_freeStreamDecode (LZ5_streamDecode_t* LZ5_stream);
/*
* LZ5_setStreamDecode
* Use this function to instruct where to find the dictionary.
* Setting a size of 0 is allowed (same effect as reset).
* Return : 1 if OK, 0 if error
*/
int LZ5_setStreamDecode (LZ5_streamDecode_t* LZ5_streamDecode, const char* dictionary, int dictSize);
/*
*_continue() :
These decoding functions allow decompression of multiple blocks in "streaming" mode.
Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
In the case of a ring buffers, decoding buffer must be either :
- Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
- Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including small ones ( < 64 KB).
- _At least_ 64 KB + 8 bytes + maxBlockSize.
In which case, encoding and decoding buffers do not need to be synchronized,
and encoding ring buffer can have any size, including larger than decoding buffer.
Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
and indicate where it is saved using LZ5_setStreamDecode()
*/
int LZ5_decompress_safe_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
int LZ5_decompress_fast_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int originalSize);
/*
Advanced decoding functions :
*_usingDict() :
These decoding functions work the same as
a combination of LZ5_setStreamDecode() followed by LZ5_decompress_x_continue()
They are stand-alone. They don't need nor update an LZ5_streamDecode_t structure.
*/
int LZ5_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
int LZ5_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
/**************************************
* Obsolete Functions
**************************************/
/* Deprecate Warnings */
/* Should these warnings messages be a problem,
it is generally possible to disable them,
with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual for example.
Otherwise, you can also define LZ5_DISABLE_DEPRECATE_WARNINGS */
#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#ifdef LZ5_DISABLE_DEPRECATE_WARNINGS
# define LZ5_DEPRECATED() /* disable deprecation warnings */
#else
# if (LZ5_GCC_VERSION >= 405) || defined(__clang__)
# define LZ5_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ5_GCC_VERSION >= 301)
# define LZ5_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ5_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ5_DEPRECATED for this compiler")
# define LZ5_DEPRECATED(message)
# endif
#endif /* LZ5_DISABLE_DEPRECATE_WARNINGS */
/* Obsolete compression functions */
/* These functions will generate warnings in a future release */
int LZ5_compress (const char* source, char* dest, int sourceSize);
int LZ5_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
int LZ5_compress_withState (void* state, const char* source, char* dest, int inputSize);
int LZ5_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ5_compress_continue (LZ5_stream_t* LZ5_streamPtr, const char* source, char* dest, int inputSize);
int LZ5_compress_limitedOutput_continue (LZ5_stream_t* LZ5_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
/* Obsolete decompression functions */
/* These function names are completely deprecated and must no longer be used.
They are only provided in lz5.c for compatibility with older programs.
- LZ5_uncompress is the same as LZ5_decompress_fast
- LZ5_uncompress_unknownOutputSize is the same as LZ5_decompress_safe
These function prototypes are now disabled; uncomment them only if you really need them.
It is highly recommended to stop using these prototypes and migrate to maintained ones */
/* int LZ5_uncompress (const char* source, char* dest, int outputSize); */
/* int LZ5_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
/* Obsolete streaming functions; use new streaming interface whenever possible */
LZ5_DEPRECATED("use LZ5_createStream() instead") void* LZ5_create (char* inputBuffer);
LZ5_DEPRECATED("use LZ5_createStream() instead") int LZ5_sizeofStreamState(void);
LZ5_DEPRECATED("use LZ5_resetStream() instead") int LZ5_resetStreamState(void* state, char* inputBuffer);
LZ5_DEPRECATED("use LZ5_saveDict() instead") char* LZ5_slideInputBuffer (void* state);
/* Obsolete streaming decoding functions */
LZ5_DEPRECATED("use LZ5_decompress_safe_usingDict() instead") int LZ5_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
LZ5_DEPRECATED("use LZ5_decompress_fast_usingDict() instead") int LZ5_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
#if defined (__cplusplus)
}
#endif

330
C/lz5/lz5common.h Normal file
View File

@@ -0,0 +1,330 @@
#ifndef LZ5COMMON_H
#define LZ5COMMON_H
#if defined (__cplusplus)
extern "C" {
#endif
/**************************************
* Tuning parameters
**************************************/
/*
* HEAPMODE :
* Select how default compression functions will allocate memory for their hash table,
* in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
*/
#ifdef _MSC_VER
#define HEAPMODE 1 /* Default stack size for VC++ is 1 MB and size of LZ5_stream_t exceeds that limit */
#else
#define HEAPMODE 0
#endif
/*
* ACCELERATION_DEFAULT :
* Select "acceleration" for LZ5_compress_fast() when parameter value <= 0
*/
#define ACCELERATION_DEFAULT 1
/**************************************
* Compiler Options
**************************************/
#ifdef _MSC_VER /* Visual Studio */
# define FORCE_INLINE static __forceinline
# include <intrin.h>
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4293) /* disable: C4293: too large shift (32-bits) */
#else
# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */
# if defined(__GNUC__) || defined(__clang__)
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif /* _MSC_VER */
#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
#if (LZ5_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
# define expect(expr,value) (expr)
#endif
#define likely(expr) expect((expr) != 0, 1)
#define unlikely(expr) expect((expr) != 0, 0)
/**************************************
* Memory routines
**************************************/
#include <stdlib.h> /* malloc, calloc, free */
#define ALLOCATOR(n,s) calloc(n,s)
#define FREEMEM free
#include <string.h> /* memset, memcpy */
#define MEM_INIT memset
/**************************************
* Common Constants
**************************************/
#define MINMATCH 3 // should be 3 or 4
#define WILDCOPYLENGTH 8
#define LASTLITERALS 5
#define MFLIMIT (WILDCOPYLENGTH+MINMATCH)
static const int LZ5_minLength = (MFLIMIT+1);
#define KB *(1 <<10)
#define MB *(1 <<20)
#define GB *(1U<<30)
#define MAXD_LOG 22
#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
#define LZ5_DICT_SIZE (1 << MAXD_LOG)
#define ML_BITS 3
#define ML_MASK ((1U<<ML_BITS)-1)
#define RUN_BITS 3
#define RUN_MASK ((1U<<RUN_BITS)-1)
#define RUN_BITS2 2
#define RUN_MASK2 ((1U<<RUN_BITS2)-1)
#define ML_RUN_BITS (ML_BITS + RUN_BITS)
#define ML_RUN_BITS2 (ML_BITS + RUN_BITS2)
#define LZ5_SHORT_OFFSET_BITS 10
#define LZ5_SHORT_OFFSET_DISTANCE (1<<LZ5_SHORT_OFFSET_BITS)
#define LZ5_MID_OFFSET_BITS 16
#define LZ5_MID_OFFSET_DISTANCE (1<<LZ5_MID_OFFSET_BITS)
/**************************************
* Common Utils
**************************************/
#define LZ5_STATIC_ASSERT(c) { enum { LZ5_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */
/****************************************************************
* Basic Types
*****************************************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef int16_t S16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef signed short S16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
#endif
/* *************************************
* HC Inline functions and Macros
***************************************/
#include "mem.h" // MEM_read
#include "lz5.h" // LZ5HC_MAX_CLEVEL
static const U32 prime4bytes = 2654435761U;
static const U64 prime5bytes = 889523592379ULL;
#ifdef LZ5HC_INCLUDES
static const U32 prime3bytes = 506832829U;
static const U64 prime6bytes = 227718039650203ULL;
static const U64 prime7bytes = 58295818150454627ULL;
static U32 LZ5HC_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; }
static size_t LZ5HC_hash3Ptr(const void* ptr, U32 h) { return LZ5HC_hash3(MEM_read32(ptr), h); }
static U32 LZ5HC_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
static size_t LZ5HC_hash4Ptr(const void* ptr, U32 h) { return LZ5HC_hash4(MEM_read32(ptr), h); }
static size_t LZ5HC_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; }
static size_t LZ5HC_hash5Ptr(const void* p, U32 h) { return LZ5HC_hash5(MEM_read64(p), h); }
static size_t LZ5HC_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
static size_t LZ5HC_hash6Ptr(const void* p, U32 h) { return LZ5HC_hash6(MEM_read64(p), h); }
static size_t LZ5HC_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; }
static size_t LZ5HC_hash7Ptr(const void* p, U32 h) { return LZ5HC_hash7(MEM_read64(p), h); }
static size_t LZ5HC_hashPtr(const void* p, U32 hBits, U32 mls)
{
switch(mls)
{
default:
case 4: return LZ5HC_hash4Ptr(p, hBits);
case 5: return LZ5HC_hash5Ptr(p, hBits);
case 6: return LZ5HC_hash6Ptr(p, hBits);
case 7: return LZ5HC_hash7Ptr(p, hBits);
}
}
/**************************************
* HC Local Macros
**************************************/
#define LZ5HC_DEBUG(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ5_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ5_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define LZ5_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__)
#define MAX(a,b) ((a)>(b))?(a):(b)
#define LZ5_OPT_NUM (1<<12)
#define LZ5_SHORT_LITERALS ((1<<RUN_BITS2)-1)
#define LZ5_LITERALS ((1<<RUN_BITS)-1)
#define LZ5_SHORT_LITLEN_COST(len) (len<LZ5_SHORT_LITERALS ? 0 : (len-LZ5_SHORT_LITERALS < 255 ? 1 : (len-LZ5_SHORT_LITERALS-255 < (1<<7) ? 2 : 3)))
#define LZ5_LEN_COST(len) (len<LZ5_LITERALS ? 0 : (len-LZ5_LITERALS < 255 ? 1 : (len-LZ5_LITERALS-255 < (1<<7) ? 2 : 3)))
static size_t LZ5_LIT_COST(size_t len, size_t offset){ return (len)+(((offset > LZ5_MID_OFFSET_DISTANCE) || (offset<LZ5_SHORT_OFFSET_DISTANCE)) ? LZ5_SHORT_LITLEN_COST(len) : LZ5_LEN_COST(len)); }
static size_t LZ5_MATCH_COST(size_t mlen, size_t offset) { return LZ5_LEN_COST(mlen) + ((offset == 0) ? 1 : (offset<LZ5_SHORT_OFFSET_DISTANCE ? 2 : (offset<LZ5_MID_OFFSET_DISTANCE ? 3 : 4))); }
#define LZ5_CODEWORD_COST(litlen,offset,mlen) (LZ5_MATCH_COST(mlen,offset) + LZ5_LIT_COST(litlen,offset))
#define LZ5_LIT_ONLY_COST(len) ((len)+(LZ5_LEN_COST(len))+1)
#define LZ5_NORMAL_MATCH_COST(mlen,offset) (LZ5_MATCH_COST(mlen,offset))
#define LZ5_NORMAL_LIT_COST(len) (len)
FORCE_INLINE size_t LZ5HC_get_price(size_t litlen, size_t offset, size_t mlen)
{
return LZ5_CODEWORD_COST(litlen, offset, mlen);
}
FORCE_INLINE size_t LZ5HC_better_price(size_t best_off, size_t best_common, size_t off, size_t common, size_t last_off)
{
return LZ5_NORMAL_MATCH_COST(common - MINMATCH, (off == last_off) ? 0 : off) < LZ5_NORMAL_MATCH_COST(best_common - MINMATCH, (best_off == last_off) ? 0 : best_off) + (LZ5_NORMAL_LIT_COST(common - best_common) );
}
FORCE_INLINE size_t LZ5HC_more_profitable(size_t best_off, size_t best_common, size_t off, size_t common, size_t literals, size_t last_off)
{
size_t sum;
if (literals > 0)
sum = MAX(common + literals, best_common);
else
sum = MAX(common, best_common - literals);
// return LZ5_CODEWORD_COST(sum - common, (off == last_off) ? 0 : (off), common - MINMATCH) <= LZ5_CODEWORD_COST(sum - best_common, (best_off == last_off) ? 0 : (best_off), best_common - MINMATCH);
return LZ5_NORMAL_MATCH_COST(common - MINMATCH, (off == last_off) ? 0 : off) + LZ5_NORMAL_LIT_COST(sum - common) <= LZ5_NORMAL_MATCH_COST(best_common - MINMATCH, (best_off == last_off) ? 0 : (best_off)) + LZ5_NORMAL_LIT_COST(sum - best_common);
}
#endif // LZ5HC_INCLUDES
/* *************************************
* HC Types
***************************************/
/** from faster to stronger */
typedef enum { LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price, LZ5HC_optimal_price, LZ5HC_optimal_price_bt } LZ5HC_strategy;
typedef struct
{
U32 windowLog; /* largest match distance : impact decompression buffer size */
U32 contentLog; /* full search segment : larger == more compression, slower, more memory (useless for fast) */
U32 hashLog; /* dispatch table : larger == more memory, faster*/
U32 hashLog3; /* dispatch table : larger == more memory, faster*/
U32 searchNum; /* nb of searches : larger == more compression, slower*/
U32 searchLength; /* size of matches : larger == faster decompression */
U32 sufficientLength; /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */
U32 fullSearch; /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */
LZ5HC_strategy strategy;
} LZ5HC_parameters;
struct LZ5HC_Data_s
{
U32* hashTable;
U32* hashTable3;
U32* chainTable;
const BYTE* end; /* next block here to continue on current prefix */
const BYTE* base; /* All index relative to this position */
const BYTE* dictBase; /* alternate base for extDict */
const BYTE* inputBuffer; /* for debugging */
const BYTE* outputBuffer; /* for debugging */
U32 dictLimit; /* below that point, need extDict */
U32 lowLimit; /* below that point, no more dict */
U32 nextToUpdate; /* index from which to continue dictionary update */
U32 compressionLevel;
U32 last_off;
LZ5HC_parameters params;
};
typedef struct
{
int off;
int len;
int back;
} LZ5HC_match_t;
typedef struct
{
int price;
int off;
int mlen;
int litlen;
int rep;
} LZ5HC_optimal_t;
/* *************************************
* HC Pre-defined compression levels
***************************************/
static const int g_maxCompressionLevel = LZ5HC_MAX_CLEVEL;
static const int LZ5HC_compressionLevel_default = 6;
static const LZ5HC_parameters LZ5HC_defaultParameters[LZ5HC_MAX_CLEVEL+1] =
{
/* windLog, contentLog, H, H3, Snum, SL, SuffL, FS, Strategy */
{ 0, 0, 0, 0, 0, 0, 0, 0, LZ5HC_fast }, // level 0 - never used
{ MAXD_LOG, MAXD_LOG, 13, 0, 4, 6, 0, 0, LZ5HC_fast }, // level 1
{ MAXD_LOG, MAXD_LOG, 13, 0, 2, 6, 0, 0, LZ5HC_fast }, // level 2
{ MAXD_LOG, MAXD_LOG, 13, 0, 1, 5, 0, 0, LZ5HC_fast }, // level 3
{ MAXD_LOG, MAXD_LOG, 14, 13, 1, 4, 0, 0, LZ5HC_price_fast }, // level 4
{ MAXD_LOG, MAXD_LOG, 17, 13, 1, 4, 0, 0, LZ5HC_price_fast }, // level 5
{ MAXD_LOG, MAXD_LOG, 15, 13, 1, 4, 0, 0, LZ5HC_lowest_price }, // level 6
{ MAXD_LOG, MAXD_LOG, 17, 13, 1, 4, 0, 0, LZ5HC_lowest_price }, // level 7
{ MAXD_LOG, MAXD_LOG, 19, 16, 1, 4, 0, 0, LZ5HC_lowest_price }, // level 8
{ MAXD_LOG, MAXD_LOG, 23, 16, 3, 4, 0, 0, LZ5HC_lowest_price }, // level 9
{ MAXD_LOG, MAXD_LOG, 23, 16, 8, 4, 0, 0, LZ5HC_lowest_price }, // level 10
{ MAXD_LOG, MAXD_LOG, 23, 16, 8, 4, 12, 0, LZ5HC_optimal_price }, // level 11
{ MAXD_LOG, MAXD_LOG, 23, 16, 8, 4, 64, 0, LZ5HC_optimal_price }, // level 12
{ MAXD_LOG, MAXD_LOG+1, 23, 16, 8, 4, 64, 1, LZ5HC_optimal_price_bt }, // level 13
{ MAXD_LOG, MAXD_LOG+1, 23, 16, 128, 4, 64, 1, LZ5HC_optimal_price_bt }, // level 14
{ MAXD_LOG, MAXD_LOG+1, 28, 24, 1<<10, 4, 1<<10, 1, LZ5HC_optimal_price_bt }, // level 15
// { 10, 10, 10, 0, 0, 4, 0, 0, LZ5HC_fast }, // min values
// { 24, 24, 28, 24, 1<<24, 7, 1<<24, 2, LZ5HC_optimal_price }, // max values
};
#if defined (__cplusplus)
}
#endif
#endif /* LZ5COMMON_H */

1485
C/lz5/lz5frame.c Normal file
View File

File diff suppressed because it is too large Load Diff

306
C/lz5/lz5frame.h Normal file
View File

@@ -0,0 +1,306 @@
/*
LZ5 auto-framing library
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
- LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
*/
/* LZ5F is a stand-alone API to create LZ5-compressed frames
* conformant with specification v1.5.1.
* All related operations, including memory management, are handled internally by the library.
* You don't need lz5.h when using lz5frame.h.
* */
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/**************************************
* Includes
**************************************/
#include <stddef.h> /* size_t */
/**************************************
* Error management
**************************************/
typedef size_t LZ5F_errorCode_t;
unsigned LZ5F_isError(LZ5F_errorCode_t code);
const char* LZ5F_getErrorName(LZ5F_errorCode_t code); /* return error code string; useful for debugging */
/**************************************
* Frame compression types
**************************************/
//#define LZ5F_DISABLE_OBSOLETE_ENUMS
#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS
# define LZ5F_OBSOLETE_ENUM(x) ,x
#else
# define LZ5F_OBSOLETE_ENUM(x)
#endif
typedef enum {
LZ5F_default=0,
LZ5F_max64KB=1,
LZ5F_max256KB=2,
LZ5F_max1MB=3,
LZ5F_max4MB=4,
LZ5F_max16MB=5,
LZ5F_max64MB=6,
LZ5F_max256MB=7
LZ5F_OBSOLETE_ENUM(max64KB = LZ5F_max64KB)
LZ5F_OBSOLETE_ENUM(max256KB = LZ5F_max256KB)
LZ5F_OBSOLETE_ENUM(max1MB = LZ5F_max1MB)
LZ5F_OBSOLETE_ENUM(max4MB = LZ5F_max4MB)
} LZ5F_blockSizeID_t;
typedef enum {
LZ5F_blockLinked=0,
LZ5F_blockIndependent
LZ5F_OBSOLETE_ENUM(blockLinked = LZ5F_blockLinked)
LZ5F_OBSOLETE_ENUM(blockIndependent = LZ5F_blockIndependent)
} LZ5F_blockMode_t;
typedef enum {
LZ5F_noContentChecksum=0,
LZ5F_contentChecksumEnabled
LZ5F_OBSOLETE_ENUM(noContentChecksum = LZ5F_noContentChecksum)
LZ5F_OBSOLETE_ENUM(contentChecksumEnabled = LZ5F_contentChecksumEnabled)
} LZ5F_contentChecksum_t;
typedef enum {
LZ5F_frame=0,
LZ5F_skippableFrame
LZ5F_OBSOLETE_ENUM(skippableFrame = LZ5F_skippableFrame)
} LZ5F_frameType_t;
#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS
typedef LZ5F_blockSizeID_t blockSizeID_t;
typedef LZ5F_blockMode_t blockMode_t;
typedef LZ5F_frameType_t frameType_t;
typedef LZ5F_contentChecksum_t contentChecksum_t;
#endif
typedef struct {
LZ5F_blockSizeID_t blockSizeID; /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
LZ5F_blockMode_t blockMode; /* blockLinked, blockIndependent ; 0 == default */
LZ5F_contentChecksum_t contentChecksumFlag; /* noContentChecksum, contentChecksumEnabled ; 0 == default */
LZ5F_frameType_t frameType; /* LZ5F_frame, skippableFrame ; 0 == default */
unsigned long long contentSize; /* Size of uncompressed (original) content ; 0 == unknown */
unsigned reserved[2]; /* must be zero for forward compatibility */
} LZ5F_frameInfo_t;
typedef struct {
LZ5F_frameInfo_t frameInfo;
int compressionLevel; /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
unsigned autoFlush; /* 1 == always flush (reduce need for tmp buffer) */
unsigned reserved[4]; /* must be zero for forward compatibility */
} LZ5F_preferences_t;
/***********************************
* Simple compression function
***********************************/
size_t LZ5F_compressFrameBound(size_t srcSize, const LZ5F_preferences_t* preferencesPtr);
size_t LZ5F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_preferences_t* preferencesPtr);
/* LZ5F_compressFrame()
* Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.1
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ5F_compressFrameBound()
* If this condition is not respected, LZ5F_compressFrame() will fail (result is an errorCode)
* The LZ5F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
* The result of the function is the number of bytes written into dstBuffer.
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
*/
/**********************************
* Advanced compression functions
**********************************/
typedef struct LZ5F_cctx_s* LZ5F_compressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableSrc; /* 1 == src content will remain available on future calls to LZ5F_compress(); avoid saving src content within tmp buffer as future dictionary */
unsigned reserved[3];
} LZ5F_compressOptions_t;
/* Resource Management */
#define LZ5F_VERSION 100
LZ5F_errorCode_t LZ5F_createCompressionContext(LZ5F_compressionContext_t* cctxPtr, unsigned version);
LZ5F_errorCode_t LZ5F_freeCompressionContext(LZ5F_compressionContext_t cctx);
/* LZ5F_createCompressionContext() :
* The first thing to do is to create a compressionContext object, which will be used in all compression operations.
* This is achieved using LZ5F_createCompressionContext(), which takes as argument a version and an LZ5F_preferences_t structure.
* The version provided MUST be LZ5F_VERSION. It is intended to track potential version differences between different binaries.
* The function will provide a pointer to a fully allocated LZ5F_compressionContext_t object.
* If the result LZ5F_errorCode_t is not zero, there was an error during context creation.
* Object can release its memory using LZ5F_freeCompressionContext();
*/
/* Compression */
size_t LZ5F_compressBegin(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_preferences_t* prefsPtr);
/* LZ5F_compressBegin() :
* will write the frame header into dstBuffer.
* dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
* The LZ5F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
* The result of the function is the number of bytes written into dstBuffer for the header
* or an error code (can be tested using LZ5F_isError())
*/
size_t LZ5F_compressBound(size_t srcSize, const LZ5F_preferences_t* prefsPtr);
/* LZ5F_compressBound() :
* Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
* Different preferences can produce different results.
* prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
* This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
*/
size_t LZ5F_compressUpdate(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_compressOptions_t* cOptPtr);
/* LZ5F_compressUpdate()
* LZ5F_compressUpdate() can be called repetitively to compress as much data as necessary.
* The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
* You can get the minimum value of dstMaxSize by using LZ5F_compressBound().
* If this condition is not respected, LZ5F_compress() will fail (result is an errorCode).
* LZ5F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
*/
size_t LZ5F_flush(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr);
/* LZ5F_flush()
* Should you need to generate compressed data immediately, without waiting for the current block to be filled,
* you can call LZ5_flush(), which will immediately compress any remaining data buffered within cctx.
* Note that dstMaxSize must be large enough to ensure the operation will be successful.
* LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
* The result of the function is the number of bytes written into dstBuffer
* (it can be zero, this means there was no data left within cctx)
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
*/
size_t LZ5F_compressEnd(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr);
/* LZ5F_compressEnd()
* When you want to properly finish the compressed frame, just call LZ5F_compressEnd().
* It will flush whatever data remained within compressionContext (like LZ5_flush())
* but also properly finalize the frame, with an endMark and a checksum.
* The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
* The function outputs an error code if it fails (can be tested using LZ5F_isError())
* The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
* A successful call to LZ5F_compressEnd() makes cctx available again for next compression task.
*/
/***********************************
* Decompression functions
***********************************/
typedef struct LZ5F_dctx_s* LZ5F_decompressionContext_t; /* must be aligned on 8-bytes */
typedef struct {
unsigned stableDst; /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
unsigned reserved[3];
} LZ5F_decompressOptions_t;
/* Resource management */
LZ5F_errorCode_t LZ5F_createDecompressionContext(LZ5F_decompressionContext_t* dctxPtr, unsigned version);
LZ5F_errorCode_t LZ5F_freeDecompressionContext(LZ5F_decompressionContext_t dctx);
/* LZ5F_createDecompressionContext() :
* The first thing to do is to create an LZ5F_decompressionContext_t object, which will be used in all decompression operations.
* This is achieved using LZ5F_createDecompressionContext().
* The version provided MUST be LZ5F_VERSION. It is intended to track potential breaking differences between different versions.
* The function will provide a pointer to a fully allocated and initialized LZ5F_decompressionContext_t object.
* The result is an errorCode, which can be tested using LZ5F_isError().
* dctx memory can be released using LZ5F_freeDecompressionContext();
* The result of LZ5F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
* That is, it should be == 0 if decompression has been completed fully and correctly.
*/
/* Decompression */
size_t LZ5F_getFrameInfo(LZ5F_decompressionContext_t dctx,
LZ5F_frameInfo_t* frameInfoPtr,
const void* srcBuffer, size_t* srcSizePtr);
/* LZ5F_getFrameInfo()
* This function decodes frame header information (such as max blockSize, frame checksum, etc.).
* Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
* A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that.
* The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
* (note that LZ5F_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough)
* Frame header info is *copied into* an already allocated LZ5F_frameInfo_t structure.
* The function result is an hint about how many srcSize bytes LZ5F_decompress() expects for next call,
* or an error code which can be tested using LZ5F_isError()
* (typically, when there is not enough src bytes to fully decode the frame header)
* Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr)
*/
size_t LZ5F_decompress(LZ5F_decompressionContext_t dctx,
void* dstBuffer, size_t* dstSizePtr,
const void* srcBuffer, size_t* srcSizePtr,
const LZ5F_decompressOptions_t* dOptPtr);
/* LZ5F_decompress()
* Call this function repetitively to regenerate data compressed within srcBuffer.
* The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
*
* The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
*
* The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
* If number of bytes read is < number of bytes provided, then decompression operation is not completed.
* It typically happens when dstBuffer is not large enough to contain all decoded data.
* LZ5F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr)
* The function will check this condition, and refuse to continue if it is not respected.
*
* dstBuffer is supposed to be flushed between each call to the function, since its content will be overwritten.
* dst arguments can be changed at will with each consecutive call to the function.
*
* The function result is an hint of how many srcSize bytes LZ5F_decompress() expects for next call.
* Schematically, it's the size of the current (or remaining) compressed block + header of next block.
* Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
* This is just a hint, you can always provide any srcSize you want.
* When a frame is fully decoded, the function result will be 0 (no more data expected).
* If decompression failed, function result is an error code, which can be tested using LZ5F_isError().
*
* After a frame is fully decoded, dctx can be used again to decompress another frame.
*/
#if defined (__cplusplus)
}
#endif

81
C/lz5/lz5frame_static.h Normal file
View File

@@ -0,0 +1,81 @@
/*
LZ5 auto-framing library
Header File for static linking only
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
- LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/* lz5frame_static.h should be used solely in the context of static linking.
* It contains definitions which may still change overtime.
* Never use it in the context of DLL linking.
* */
/**************************************
* Includes
**************************************/
#include "lz5frame.h"
/**************************************
* Error management
* ************************************/
#define LZ5F_LIST_ERRORS(ITEM) \
ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
ITEM(ERROR_compressionLevel_invalid) \
ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
ITEM(ERROR_allocation_failed) \
ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
ITEM(ERROR_srcPtr_wrong) \
ITEM(ERROR_decompressionFailed) \
ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
ITEM(ERROR_maxCode)
//#define LZ5F_DISABLE_OLD_ENUMS
#ifndef LZ5F_DISABLE_OLD_ENUMS
#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM, ENUM = LZ5F_##ENUM,
#else
#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM,
#endif
typedef enum { LZ5F_LIST_ERRORS(LZ5F_GENERATE_ENUM) } LZ5F_errorCodes; /* enum is exposed, to handle specific errors; compare function result to -enum value */
#if defined (__cplusplus)
}
#endif

1929
C/lz5/lz5hc.c Normal file
View File

File diff suppressed because it is too large Load Diff

178
C/lz5/lz5hc.h Normal file
View File

@@ -0,0 +1,178 @@
/*
LZ5 HC - High Compression Mode of LZ5
Header File
Copyright (C) 2011-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- LZ5 source repository : https://github.com/inikep/lz5
- LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*****************************
* Includes
*****************************/
#include <stddef.h> /* size_t */
/**************************************
* Block Compression
**************************************/
int LZ5_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
/*
LZ5_compress_HC :
Destination buffer 'dst' must be already allocated.
Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible)
Worst size evaluation is provided by function LZ5_compressBound() (see "lz5.h")
srcSize : Max supported value is LZ5_MAX_INPUT_SIZE (see "lz5.h")
compressionLevel : Recommended values are between 4 and 9, although any value between 0 and LZ5HC_MAX_CLEVEL (equal to 15) will work.
0 means "use default value" (see lz5hc.c).
Values >LZ5HC_MAX_CLEVEL behave the same as LZ5HC_MAX_CLEVEL.
return : the number of bytes written into buffer 'dst'
or 0 if compression fails.
*/
/* Note :
Decompression functions are provided within LZ5 source code (see "lz5.h") (BSD license)
*/
typedef struct LZ5HC_Data_s LZ5HC_Data_Structure;
int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr, int compressionLevel);
void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr);
int LZ5_sizeofStateHC(void);
int LZ5_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize);
/*
LZ5_compress_HC_extStateHC() :
Use this function if you prefer to manually allocate memory for compression tables.
To know how much memory must be allocated for the compression tables, use :
int LZ5_sizeofStateHC();
Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
The allocated memory can then be provided to the compression functions using 'void* state' parameter.
LZ5_compress_HC_extStateHC() is equivalent to previously described function.
It just uses externally allocated memory for stateHC.
*/
/**************************************
* Streaming Compression
**************************************/
#define LZ5_STREAMHCSIZE 262192
#define LZ5_STREAMHCSIZE_SIZET (LZ5_STREAMHCSIZE / sizeof(size_t))
typedef struct { size_t table[LZ5_STREAMHCSIZE_SIZET]; } LZ5_streamHC_t;
/*
LZ5_streamHC_t
This structure allows static allocation of LZ5 HC streaming state.
State must then be initialized using LZ5_resetStreamHC() before first use.
Static allocation should only be used in combination with static linking.
If you want to use LZ5 as a DLL, please use construction functions below, which are future-proof.
*/
LZ5_streamHC_t* LZ5_createStreamHC(int compressionLevel);
int LZ5_freeStreamHC (LZ5_streamHC_t* streamHCPtr);
/*
These functions create and release memory for LZ5 HC streaming state.
Newly created states are already initialized.
Existing state space can be re-used anytime using LZ5_resetStreamHC().
If you use LZ5 as a DLL, use these functions instead of static structure allocation,
to avoid size mismatch between different versions.
*/
void LZ5_resetStreamHC (LZ5_streamHC_t* streamHCPtr);
int LZ5_loadDictHC (LZ5_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
int LZ5_compress_HC_continue (LZ5_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
int LZ5_saveDictHC (LZ5_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
/*
These functions compress data in successive blocks of any size, using previous blocks as dictionary.
One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
There is an exception for ring buffers, which can be smaller 64 KB.
Such case is automatically detected and correctly handled by LZ5_compress_HC_continue().
Before starting compression, state must be properly initialized, using LZ5_resetStreamHC().
A first "fictional block" can then be designated as initial dictionary, using LZ5_loadDictHC() (Optional).
Then, use LZ5_compress_HC_continue() to compress each successive block.
It works like LZ5_compress_HC(), but use previous memory blocks as dictionary to improve compression.
Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
As a reminder, size 'dst' buffer to handle worst cases, using LZ5_compressBound(), to ensure success of compression operation.
If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
you must save it to a safer memory space, using LZ5_saveDictHC().
Return value of LZ5_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
*/
/**************************************
* Deprecated Functions
**************************************/
/* Deprecate Warnings */
/* Should these warnings messages be a problem,
it is generally possible to disable them,
with -Wno-deprecated-declarations for gcc
or _CRT_SECURE_NO_WARNINGS in Visual for example.
You can also define LZ5_DEPRECATE_WARNING_DEFBLOCK. */
#ifndef LZ5_DEPRECATE_WARNING_DEFBLOCK
# define LZ5_DEPRECATE_WARNING_DEFBLOCK
# if (LZ5_GCC_VERSION >= 405) || defined(__clang__)
# define LZ5_DEPRECATED(message) __attribute__((deprecated(message)))
# elif (LZ5_GCC_VERSION >= 301)
# define LZ5_DEPRECATED(message) __attribute__((deprecated))
# elif defined(_MSC_VER)
# define LZ5_DEPRECATED(message) __declspec(deprecated(message))
# else
# pragma message("WARNING: You need to implement LZ5_DEPRECATED for this compiler")
# define LZ5_DEPRECATED(message)
# endif
#endif // LZ5_DEPRECATE_WARNING_DEFBLOCK
/* compression functions */
/* these functions are planned to trigger warning messages by r132 approximately */
int LZ5_compressHC (const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ5_compressHC_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
int LZ5_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
#if defined (__cplusplus)
}
#endif

915
C/lz5/lz5xxhash.c Normal file
View File

@@ -0,0 +1,915 @@
/*
xxHash - Fast Hash algorithm
Copyright (C) 2012-2015, Yann Collet
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/**************************************
* Tuning parameters
**************************************/
/* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
* For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
* If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
* You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
*/
#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
# define XXH_USE_UNALIGNED_ACCESS 1
#endif
/* XXH_ACCEPT_NULL_INPUT_POINTER :
* If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
* When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
* By default, this option is disabled. To enable it, uncomment below define :
*/
/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
/* XXH_FORCE_NATIVE_FORMAT :
* By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
* Results are therefore identical for little-endian and big-endian CPU.
* This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
* Should endian-independance be of no importance for your application, you may set the #define below to 1.
* It will improve speed for Big-endian CPU.
* This option has no impact on Little_Endian CPU.
*/
#define XXH_FORCE_NATIVE_FORMAT 0
/**************************************
* Compiler Specific Options
***************************************/
#ifdef _MSC_VER /* Visual Studio */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# define FORCE_INLINE static __forceinline
#else
# if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# ifdef __GNUC__
# define FORCE_INLINE static inline __attribute__((always_inline))
# else
# define FORCE_INLINE static inline
# endif
# else
# define FORCE_INLINE static
# endif /* __STDC_VERSION__ */
#endif
/**************************************
* Includes & Memory related functions
***************************************/
#include "xxhash.h"
/* Modify the local functions below should you wish to use some other memory routines */
/* for malloc(), free() */
#include <stdlib.h>
static void* XXH_malloc(size_t s) { return malloc(s); }
static void XXH_free (void* p) { free(p); }
/* for memcpy() */
#include <string.h>
static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
/**************************************
* Basic Types
***************************************/
#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
#endif
static U32 XXH_read32(const void* memPtr)
{
U32 val32;
memcpy(&val32, memPtr, 4);
return val32;
}
static U64 XXH_read64(const void* memPtr)
{
U64 val64;
memcpy(&val64, memPtr, 8);
return val64;
}
/******************************************
* Compiler-specific Functions and Macros
******************************************/
#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
#if defined(_MSC_VER)
# define XXH_rotl32(x,r) _rotl(x,r)
# define XXH_rotl64(x,r) _rotl64(x,r)
#else
# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
#endif
#if defined(_MSC_VER) /* Visual Studio */
# define XXH_swap32 _byteswap_ulong
# define XXH_swap64 _byteswap_uint64
#elif GCC_VERSION >= 403
# define XXH_swap32 __builtin_bswap32
# define XXH_swap64 __builtin_bswap64
#else
static U32 XXH_swap32 (U32 x)
{
return ((x << 24) & 0xff000000 ) |
((x << 8) & 0x00ff0000 ) |
((x >> 8) & 0x0000ff00 ) |
((x >> 24) & 0x000000ff );
}
static U64 XXH_swap64 (U64 x)
{
return ((x << 56) & 0xff00000000000000ULL) |
((x << 40) & 0x00ff000000000000ULL) |
((x << 24) & 0x0000ff0000000000ULL) |
((x << 8) & 0x000000ff00000000ULL) |
((x >> 8) & 0x00000000ff000000ULL) |
((x >> 24) & 0x0000000000ff0000ULL) |
((x >> 40) & 0x000000000000ff00ULL) |
((x >> 56) & 0x00000000000000ffULL);
}
#endif
/***************************************
* Architecture Macros
***************************************/
typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
#ifndef XXH_CPU_LITTLE_ENDIAN /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */
static const int one = 1;
# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&one))
#endif
/*****************************
* Memory reads
*****************************/
typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
else
return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
}
FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
{
return XXH_readLE32_align(ptr, endian, XXH_unaligned);
}
FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
{
if (align==XXH_unaligned)
return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
else
return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
}
FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
{
return XXH_readLE64_align(ptr, endian, XXH_unaligned);
}
/***************************************
* Macros
***************************************/
#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } /* use only *after* variable declarations */
/***************************************
* Constants
***************************************/
#define PRIME32_1 2654435761U
#define PRIME32_2 2246822519U
#define PRIME32_3 3266489917U
#define PRIME32_4 668265263U
#define PRIME32_5 374761393U
#define PRIME64_1 11400714785074694791ULL
#define PRIME64_2 14029467366897019727ULL
#define PRIME64_3 1609587929392839161ULL
#define PRIME64_4 9650029242287828579ULL
#define PRIME64_5 2870177450012600261ULL
/*****************************
* Simple Hash Functions
*****************************/
FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
U32 h32;
#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL)
{
len=0;
bEnd=p=(const BYTE*)(size_t)16;
}
#endif
if (len>=16)
{
const BYTE* const limit = bEnd - 16;
U32 v1 = seed + PRIME32_1 + PRIME32_2;
U32 v2 = seed + PRIME32_2;
U32 v3 = seed + 0;
U32 v4 = seed - PRIME32_1;
do
{
v1 += XXH_get32bits(p) * PRIME32_2;
v1 = XXH_rotl32(v1, 13);
v1 *= PRIME32_1;
p+=4;
v2 += XXH_get32bits(p) * PRIME32_2;
v2 = XXH_rotl32(v2, 13);
v2 *= PRIME32_1;
p+=4;
v3 += XXH_get32bits(p) * PRIME32_2;
v3 = XXH_rotl32(v3, 13);
v3 *= PRIME32_1;
p+=4;
v4 += XXH_get32bits(p) * PRIME32_2;
v4 = XXH_rotl32(v4, 13);
v4 *= PRIME32_1;
p+=4;
}
while (p<=limit);
h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
}
else
{
h32 = seed + PRIME32_5;
}
h32 += (U32) len;
while (p+4<=bEnd)
{
h32 += XXH_get32bits(p) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4 ;
p+=4;
}
while (p<bEnd)
{
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
p++;
}
h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
h32 *= PRIME32_3;
h32 ^= h32 >> 16;
return h32;
}
unsigned XXH32 (const void* input, size_t len, unsigned seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH32_state_t state;
XXH32_reset(&state, seed);
XXH32_update(&state, input, len);
return XXH32_digest(&state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 3) == 0) /* Input is 4-bytes aligned, leverage the speed benefit */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
}
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
{
const BYTE* p = (const BYTE*)input;
const BYTE* bEnd = p + len;
U64 h64;
#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (p==NULL)
{
len=0;
bEnd=p=(const BYTE*)(size_t)32;
}
#endif
if (len>=32)
{
const BYTE* const limit = bEnd - 32;
U64 v1 = seed + PRIME64_1 + PRIME64_2;
U64 v2 = seed + PRIME64_2;
U64 v3 = seed + 0;
U64 v4 = seed - PRIME64_1;
do
{
v1 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
v2 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
v3 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
v4 += XXH_get64bits(p) * PRIME64_2;
p+=8;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
}
while (p<=limit);
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
v1 *= PRIME64_2;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
h64 ^= v1;
h64 = h64 * PRIME64_1 + PRIME64_4;
v2 *= PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
h64 ^= v2;
h64 = h64 * PRIME64_1 + PRIME64_4;
v3 *= PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
h64 ^= v3;
h64 = h64 * PRIME64_1 + PRIME64_4;
v4 *= PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
h64 ^= v4;
h64 = h64 * PRIME64_1 + PRIME64_4;
}
else
{
h64 = seed + PRIME64_5;
}
h64 += (U64) len;
while (p+8<=bEnd)
{
U64 k1 = XXH_get64bits(p);
k1 *= PRIME64_2;
k1 = XXH_rotl64(k1,31);
k1 *= PRIME64_1;
h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
if (p+4<=bEnd)
{
h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
while (p<bEnd)
{
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
{
#if 0
/* Simple version, good for code maintenance, but unfortunately slow for small inputs */
XXH64_state_t state;
XXH64_reset(&state, seed);
XXH64_update(&state, input, len);
return XXH64_digest(&state);
#else
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
# if !defined(XXH_USE_UNALIGNED_ACCESS)
if ((((size_t)input) & 7)==0) /* Input is aligned, let's leverage the speed advantage */
{
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
}
# endif
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
else
return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
#endif
}
/****************************************************
* Advanced Hash Functions
****************************************************/
/*** Allocation ***/
typedef struct
{
U64 total_len;
U32 seed;
U32 v1;
U32 v2;
U32 v3;
U32 v4;
U32 mem32[4]; /* defined as U32 for alignment */
U32 memsize;
} XXH_istate32_t;
typedef struct
{
U64 total_len;
U64 seed;
U64 v1;
U64 v2;
U64 v3;
U64 v4;
U64 mem64[4]; /* defined as U64 for alignment */
U32 memsize;
} XXH_istate64_t;
XXH32_state_t* XXH32_createState(void)
{
XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t)); /* A compilation error here means XXH32_state_t is not large enough */
return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
}
XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
XXH64_state_t* XXH64_createState(void)
{
XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t)); /* A compilation error here means XXH64_state_t is not large enough */
return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
}
XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
{
XXH_free(statePtr);
return XXH_OK;
}
/*** Hash feed ***/
XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed)
{
XXH_istate32_t* state = (XXH_istate32_t*) state_in;
state->seed = seed;
state->v1 = seed + PRIME32_1 + PRIME32_2;
state->v2 = seed + PRIME32_2;
state->v3 = seed + 0;
state->v4 = seed - PRIME32_1;
state->total_len = 0;
state->memsize = 0;
return XXH_OK;
}
XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed)
{
XXH_istate64_t* state = (XXH_istate64_t*) state_in;
state->seed = seed;
state->v1 = seed + PRIME64_1 + PRIME64_2;
state->v2 = seed + PRIME64_2;
state->v3 = seed + 0;
state->v4 = seed - PRIME64_1;
state->total_len = 0;
state->memsize = 0;
return XXH_OK;
}
FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
{
XXH_istate32_t* state = (XXH_istate32_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
if (state->memsize + len < 16) /* fill in tmp buffer */
{
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
if (state->memsize) /* some data left from previous update */
{
XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
{
const U32* p32 = state->mem32;
state->v1 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v1 = XXH_rotl32(state->v1, 13);
state->v1 *= PRIME32_1;
p32++;
state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v2 = XXH_rotl32(state->v2, 13);
state->v2 *= PRIME32_1;
p32++;
state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v3 = XXH_rotl32(state->v3, 13);
state->v3 *= PRIME32_1;
p32++;
state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
state->v4 = XXH_rotl32(state->v4, 13);
state->v4 *= PRIME32_1;
p32++;
}
p += 16-state->memsize;
state->memsize = 0;
}
if (p <= bEnd-16)
{
const BYTE* const limit = bEnd - 16;
U32 v1 = state->v1;
U32 v2 = state->v2;
U32 v3 = state->v3;
U32 v4 = state->v4;
do
{
v1 += XXH_readLE32(p, endian) * PRIME32_2;
v1 = XXH_rotl32(v1, 13);
v1 *= PRIME32_1;
p+=4;
v2 += XXH_readLE32(p, endian) * PRIME32_2;
v2 = XXH_rotl32(v2, 13);
v2 *= PRIME32_1;
p+=4;
v3 += XXH_readLE32(p, endian) * PRIME32_2;
v3 = XXH_rotl32(v3, 13);
v3 *= PRIME32_1;
p+=4;
v4 += XXH_readLE32(p, endian) * PRIME32_2;
v4 = XXH_rotl32(v4, 13);
v4 *= PRIME32_1;
p+=4;
}
while (p<=limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd)
{
XXH_memcpy(state->mem32, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
return XXH_OK;
}
XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
else
return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
{
const XXH_istate32_t* state = (const XXH_istate32_t*) state_in;
const BYTE * p = (const BYTE*)state->mem32;
const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
U32 h32;
if (state->total_len >= 16)
{
h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
}
else
{
h32 = state->seed + PRIME32_5;
}
h32 += (U32) state->total_len;
while (p+4<=bEnd)
{
h32 += XXH_readLE32(p, endian) * PRIME32_3;
h32 = XXH_rotl32(h32, 17) * PRIME32_4;
p+=4;
}
while (p<bEnd)
{
h32 += (*p) * PRIME32_5;
h32 = XXH_rotl32(h32, 11) * PRIME32_1;
p++;
}
h32 ^= h32 >> 15;
h32 *= PRIME32_2;
h32 ^= h32 >> 13;
h32 *= PRIME32_3;
h32 ^= h32 >> 16;
return h32;
}
U32 XXH32_digest (const XXH32_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH32_digest_endian(state_in, XXH_littleEndian);
else
return XXH32_digest_endian(state_in, XXH_bigEndian);
}
FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
{
XXH_istate64_t * state = (XXH_istate64_t *) state_in;
const BYTE* p = (const BYTE*)input;
const BYTE* const bEnd = p + len;
#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
if (input==NULL) return XXH_ERROR;
#endif
state->total_len += len;
if (state->memsize + len < 32) /* fill in tmp buffer */
{
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
state->memsize += (U32)len;
return XXH_OK;
}
if (state->memsize) /* some data left from previous update */
{
XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
{
const U64* p64 = state->mem64;
state->v1 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v1 = XXH_rotl64(state->v1, 31);
state->v1 *= PRIME64_1;
p64++;
state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v2 = XXH_rotl64(state->v2, 31);
state->v2 *= PRIME64_1;
p64++;
state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v3 = XXH_rotl64(state->v3, 31);
state->v3 *= PRIME64_1;
p64++;
state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
state->v4 = XXH_rotl64(state->v4, 31);
state->v4 *= PRIME64_1;
p64++;
}
p += 32-state->memsize;
state->memsize = 0;
}
if (p+32 <= bEnd)
{
const BYTE* const limit = bEnd - 32;
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
do
{
v1 += XXH_readLE64(p, endian) * PRIME64_2;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
p+=8;
v2 += XXH_readLE64(p, endian) * PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
p+=8;
v3 += XXH_readLE64(p, endian) * PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
p+=8;
v4 += XXH_readLE64(p, endian) * PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
p+=8;
}
while (p<=limit);
state->v1 = v1;
state->v2 = v2;
state->v3 = v3;
state->v4 = v4;
}
if (p < bEnd)
{
XXH_memcpy(state->mem64, p, bEnd-p);
state->memsize = (int)(bEnd-p);
}
return XXH_OK;
}
XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
else
return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
}
FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
{
const XXH_istate64_t * state = (const XXH_istate64_t *) state_in;
const BYTE * p = (const BYTE*)state->mem64;
const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
U64 h64;
if (state->total_len >= 32)
{
U64 v1 = state->v1;
U64 v2 = state->v2;
U64 v3 = state->v3;
U64 v4 = state->v4;
h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
v1 *= PRIME64_2;
v1 = XXH_rotl64(v1, 31);
v1 *= PRIME64_1;
h64 ^= v1;
h64 = h64*PRIME64_1 + PRIME64_4;
v2 *= PRIME64_2;
v2 = XXH_rotl64(v2, 31);
v2 *= PRIME64_1;
h64 ^= v2;
h64 = h64*PRIME64_1 + PRIME64_4;
v3 *= PRIME64_2;
v3 = XXH_rotl64(v3, 31);
v3 *= PRIME64_1;
h64 ^= v3;
h64 = h64*PRIME64_1 + PRIME64_4;
v4 *= PRIME64_2;
v4 = XXH_rotl64(v4, 31);
v4 *= PRIME64_1;
h64 ^= v4;
h64 = h64*PRIME64_1 + PRIME64_4;
}
else
{
h64 = state->seed + PRIME64_5;
}
h64 += (U64) state->total_len;
while (p+8<=bEnd)
{
U64 k1 = XXH_readLE64(p, endian);
k1 *= PRIME64_2;
k1 = XXH_rotl64(k1,31);
k1 *= PRIME64_1;
h64 ^= k1;
h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
p+=8;
}
if (p+4<=bEnd)
{
h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
p+=4;
}
while (p<bEnd)
{
h64 ^= (*p) * PRIME64_5;
h64 = XXH_rotl64(h64, 11) * PRIME64_1;
p++;
}
h64 ^= h64 >> 33;
h64 *= PRIME64_2;
h64 ^= h64 >> 29;
h64 *= PRIME64_3;
h64 ^= h64 >> 32;
return h64;
}
unsigned long long XXH64_digest (const XXH64_state_t* state_in)
{
XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
return XXH64_digest_endian(state_in, XXH_littleEndian);
else
return XXH64_digest_endian(state_in, XXH_bigEndian);
}

452
C/lz5/mem.h Normal file
View File

@@ -0,0 +1,452 @@
/* ******************************************************************
mem.h
low-level memory access routines
Copyright (C) 2013-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
- Public forum : https://groups.google.com/forum/#!forum/lz4c
****************************************************************** */
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/******************************************
* Includes
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */
/******************************************
* Compiler-specific
******************************************/
#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
# define MEM_STATIC static __inline
#elif defined(__GNUC__)
# define MEM_STATIC static __attribute__((unused))
#else
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/****************************************************************
* Memory I/O
*****************************************************************/
/* MEM_FORCE_MEMORY_ACCESS
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets generating assembly depending on alignment.
* But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
#endif
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
MEM_STATIC unsigned MEM_isLittleEndian(void)
{
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
}
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
/* violates C standard on structure alignment.
Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
/* default method, safe and standard.
can sometimes prove slower */
MEM_STATIC U16 MEM_read16(const void* memPtr)
{
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U32 MEM_read32(const void* memPtr)
{
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U64 MEM_read64(const void* memPtr)
{
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif // MEM_FORCE_MEMORY_ACCESS
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read16(memPtr);
else
{
const BYTE* p = (const BYTE*)memPtr;
return (U16)(p[0] + (p[1]<<8));
}
}
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
{
if (MEM_isLittleEndian())
{
MEM_write16(memPtr, val);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val;
p[1] = (BYTE)(val>>8);
}
}
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{
if (MEM_isLittleEndian())
{
U32 val32 = 0;
memcpy(&val32, memPtr, 3);
return val32;
}
else
{
const BYTE* p = (const BYTE*)memPtr;
return (U32)(p[0] + (p[1]<<8) + (p[2]<<16));
}
}
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 value)
{
if (MEM_isLittleEndian())
{
memcpy(memPtr, &value, 3);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE) value;
p[1] = (BYTE)(value>>8);
p[2] = (BYTE)(value>>16);
}
}
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read32(memPtr);
else
{
const BYTE* p = (const BYTE*)memPtr;
return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
}
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
{
MEM_write32(memPtr, val32);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val32;
p[1] = (BYTE)(val32>>8);
p[2] = (BYTE)(val32>>16);
p[3] = (BYTE)(val32>>24);
}
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read64(memPtr);
else
{
const BYTE* p = (const BYTE*)memPtr;
return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
+ ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
}
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
{
MEM_write64(memPtr, val64);
}
else
{
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val64;
p[1] = (BYTE)(val64>>8);
p[2] = (BYTE)(val64>>16);
p[3] = (BYTE)(val64>>24);
p[4] = (BYTE)(val64>>32);
p[5] = (BYTE)(val64>>40);
p[6] = (BYTE)(val64>>48);
p[7] = (BYTE)(val64>>56);
}
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readLE32(memPtr);
else
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
#if MINMATCH == 3
#define MEM_read24(ptr) (U32)(MEM_read32(ptr)<<8)
#else
#define MEM_read24(ptr) (U32)(MEM_read32(ptr))
#endif
/* **************************************
* Function body to include for inlining
****************************************/
static size_t MEM_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
#define MIN(a,b) ((a)<(b) ? (a) : (b))
/*static unsigned MEM_highbit(U32 val)
{
# if defined(_MSC_VER) // Visual
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) // GCC Intrinsic
return 31 - __builtin_clz(val);
# else // Software version
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}*/
MEM_STATIC unsigned MEM_NbCommonBytes (register size_t val)
{
if (MEM_isLittleEndian())
{
if (MEM_64bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64( &r, (U64)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER)
unsigned long r=0;
_BitScanForward( &r, (U32)val );
return (int)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else /* Big Endian CPU */
{
if (MEM_32bits())
{
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanReverse64( &r, val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
r += (!val);
return r;
# endif
}
else /* 32 bits */
{
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanReverse( &r, (unsigned long)val );
return (unsigned)(r>>3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t MEM_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
{
const BYTE* const pStart = pIn;
while ((pIn<pInLimit-(sizeof(size_t)-1)))
{
size_t diff = MEM_read_ARCH(pMatch) ^ MEM_read_ARCH(pIn);
if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
pIn += MEM_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
static void MEM_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
#define COPY8(d,s) { MEM_copy8(d,s); d+=8; s+=8; }
/*! MEM_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
/*static void MEM_wildcopy(void* dst, const void* src, size_t length)
{
const BYTE* ip = (const BYTE*)src;
BYTE* op = (BYTE*)dst;
BYTE* const oend = op + length;
do
COPY8(op, ip)
while (op < oend);
} */
/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */
static void MEM_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
{
BYTE* d = (BYTE*)dstPtr;
const BYTE* s = (const BYTE*)srcPtr;
BYTE* const e = (BYTE*)dstEnd;
do { MEM_copy8(d,s); d+=8; s+=8; } while (d<e);
}
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */

192
C/lz5/xxhash.h Normal file
View File

@@ -0,0 +1,192 @@
/*
xxHash - Extremely Fast Hash algorithm
Header File
Copyright (C) 2012-2015, Yann Collet.
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
You can contact the author at :
- xxHash source repository : https://github.com/Cyan4973/xxHash
*/
/* Notice extracted from xxHash homepage :
xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
It also successfully passes all tests from the SMHasher suite.
Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
Name Speed Q.Score Author
xxHash 5.4 GB/s 10
CrapWow 3.2 GB/s 2 Andrew
MumurHash 3a 2.7 GB/s 10 Austin Appleby
SpookyHash 2.0 GB/s 10 Bob Jenkins
SBox 1.4 GB/s 9 Bret Mulvey
Lookup3 1.2 GB/s 9 Bob Jenkins
SuperFastHash 1.2 GB/s 1 Paul Hsieh
CityHash64 1.05 GB/s 10 Pike & Alakuijala
FNV 0.55 GB/s 5 Fowler, Noll, Vo
CRC32 0.43 GB/s 9
MD5-32 0.33 GB/s 10 Ronald L. Rivest
SHA1-32 0.28 GB/s 10
Q.Score is a measure of quality of the hash function.
It depends on successfully passing SMHasher test set.
10 is a perfect score.
A 64-bits version, named XXH64, is available since r35.
It offers much better speed, but for 64-bits applications only.
Name Speed on 64 bits Speed on 32 bits
XXH64 13.8 GB/s 1.9 GB/s
XXH32 6.8 GB/s 6.0 GB/s
*/
#pragma once
#if defined (__cplusplus)
extern "C" {
#endif
/*****************************
* Definitions
*****************************/
#include <stddef.h> /* size_t */
typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
/*****************************
* Namespace Emulation
*****************************/
/* Motivations :
If you need to include xxHash into your library,
but wish to avoid xxHash symbols to be present on your library interface
in an effort to avoid potential name collision if another library also includes xxHash,
you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash
with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values).
Note that no change is required within the calling program :
it can still call xxHash functions using their regular name.
They will be automatically translated by this header.
*/
#ifdef XXH_NAMESPACE
# define XXH_CAT(A,B) A##B
# define XXH_NAME2(A,B) XXH_CAT(A,B)
# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
#endif
/*****************************
* Simple Hash Functions
*****************************/
unsigned int XXH32 (const void* input, size_t length, unsigned seed);
unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
/*
XXH32() :
Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
The memory between input & input+length must be valid (allocated and read-accessible).
"seed" can be used to alter the result predictably.
This function successfully passes all SMHasher tests.
Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
XXH64() :
Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
Faster on 64-bits systems. Slower on 32-bits systems.
*/
/*****************************
* Advanced Hash Functions
*****************************/
typedef struct { long long ll[ 6]; } XXH32_state_t;
typedef struct { long long ll[11]; } XXH64_state_t;
/*
These structures allow static allocation of XXH states.
States must then be initialized using XXHnn_reset() before first use.
If you prefer dynamic allocation, please refer to functions below.
*/
XXH32_state_t* XXH32_createState(void);
XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
XXH64_state_t* XXH64_createState(void);
XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
/*
These functions create and release memory for XXH state.
States must then be initialized using XXHnn_reset() before first use.
*/
XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned seed);
XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
unsigned int XXH32_digest (const XXH32_state_t* statePtr);
XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed);
XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
/*
These functions calculate the xxHash of an input provided in multiple smaller packets,
as opposed to an input provided as a single block.
XXH state space must first be allocated, using either static or dynamic method provided above.
Start a new hash by initializing state with a seed, using XXHnn_reset().
Then, feed the hash state by calling XXHnn_update() as many times as necessary.
Obviously, input must be valid, meaning allocated and read accessible.
The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
Finally, you can produce a hash anytime, by using XXHnn_digest().
This function returns the final nn-bits hash.
You can nonetheless continue feeding the hash state with more input,
and therefore get some new hashes, by calling again XXHnn_digest().
When you are done, don't forget to free XXH state space, using typically XXHnn_freeState().
*/
#if defined (__cplusplus)
}
#endif

43
C/zstd/error_private.c Normal file
View File

@@ -0,0 +1,43 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
/* The purpose of this file is to have a single list of error strings embedded in binary */
#include "error_private.h"
const char* ERR_getErrorString(ERR_enum code)
{
static const char* const notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(version_unsupported): return "Version not supported";
case PREFIX(parameter_unknown): return "Unknown parameter type";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
}

32
C/zstd/error_private.h
View File

@@ -21,7 +21,7 @@ extern "C" {
* Dependencies
******************************************/
#include <stddef.h> /* size_t */
#include "error_public.h" /* enum list */
#include "zstd_errors.h" /* enum list */
/* ****************************************
@@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu
* Error Strings
******************************************/
ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
{
static const char* notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(prefix_unknown): return "Unknown frame descriptor";
case PREFIX(version_unsupported): return "Version not supported";
case PREFIX(parameter_unknown): return "Unknown parameter type";
case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size incorrect";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
case PREFIX(dictionary_wrong): return "Dictionary mismatch";
case PREFIX(maxCode):
default: return notErrorCode;
}
}
const char* ERR_getErrorString(ERR_enum code); /* error_private.c */
ERR_STATIC const char* ERR_getErrorName(size_t code)
{

8
C/zstd/fse.h
View File

@@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
BIT_flushBits(bitC);
}
/* ====== Decompression ====== */
typedef struct {
@@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
* Increasing memory usage improves compression ratio
* Reduced memory usage can improve speed, due to cache effect
* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
#ifndef FSE_MAX_MEMORY_USAGE
# define FSE_MAX_MEMORY_USAGE 14
#endif
#ifndef FSE_DEFAULT_MEMORY_USAGE
# define FSE_DEFAULT_MEMORY_USAGE 13
#endif
/*!FSE_MAX_SYMBOL_VALUE :
* Maximum symbol value authorized.
* Required for proper stack allocation */
#ifndef FSE_MAX_SYMBOL_VALUE
# define FSE_MAX_SYMBOL_VALUE 255
#endif
/* **************************************************************
* template functions type & suffix

2
C/zstd/zstd.h
View File

@@ -33,7 +33,7 @@ extern "C" {
/*======= Version =======*/
#define ZSTD_VERSION_MAJOR 1
#define ZSTD_VERSION_MINOR 1
#define ZSTD_VERSION_RELEASE 0
#define ZSTD_VERSION_RELEASE 1
#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
#define ZSTD_QUOTE(str) #str

6
C/zstd/zstd_compress.c
View File

@@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
const U32 dictLimit = zc->dictLimit;
const BYTE* const dictEnd = dictBase + dictLimit;
const BYTE* const prefixStart = base + dictLimit;
const BYTE* match = base + matchIndex;
const BYTE* match;
const U32 current = (U32)(ip-base);
const U32 btLow = btMask >= current ? 0 : current - btMask;
U32* smallerPtr = bt + 2*(current&btMask);
@@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
BYTE* op = ostart;
U32 const maxDist = 1 << cctx->params.cParams.windowLog;
if (cctx->params.fParams.checksumFlag)
if (cctx->params.fParams.checksumFlag && srcSize)
XXH64_update(&cctx->xxhState, src, srcSize);
while (remaining) {
@@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
return NULL;
}
if (dictSize) {
memcpy(dictContent, dict, dictSize);
}
{ size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);

15
C/zstd/zstd_decompress.c
View File

@@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
if (!singleSegment) {
BYTE const wlByte = ip[pos++];
U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */
windowSize = (1U << windowLog);
windowSize += (windowSize >> 3) * (wlByte&7);
}
@@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
}
if (!windowSize) windowSize = (U32)frameContentSize;
if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
fparamsPtr->frameContentSize = frameContentSize;
fparamsPtr->windowSize = windowSize;
fparamsPtr->dictID = dictID;
@@ -878,7 +878,12 @@ size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {
@@ -1397,7 +1402,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
return NULL;
}
if (dictSize) {
memcpy(dictContent, dict, dictSize);
}
{ size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
if (ZSTD_isError(errorCode)) {
ZSTD_free(dictContent, customMem);
@@ -1568,7 +1575,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
switch(paramType)
{
default : return ERROR(parameter_unknown);
case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
}
return 0;
}
@@ -1649,7 +1656,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
} }
zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
/* Adapt buffer sizes to frame header instructions */
{ size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);

60
C/zstd/zstd_errors.h Normal file
View File

@@ -0,0 +1,60 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#ifndef ZSTD_ERRORS_H_398273423
#define ZSTD_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
#endif
/*===== dependency =====*/
#include <stddef.h> /* size_t */
/*-****************************************
* error codes list
******************************************/
typedef enum {
ZSTD_error_no_error,
ZSTD_error_GENERIC,
ZSTD_error_prefix_unknown,
ZSTD_error_version_unsupported,
ZSTD_error_parameter_unknown,
ZSTD_error_frameParameter_unsupported,
ZSTD_error_frameParameter_unsupportedBy32bits,
ZSTD_error_frameParameter_windowTooLarge,
ZSTD_error_compressionParameter_unsupported,
ZSTD_error_init_missing,
ZSTD_error_memory_allocation,
ZSTD_error_stage_wrong,
ZSTD_error_dstSize_tooSmall,
ZSTD_error_srcSize_wrong,
ZSTD_error_corruption_detected,
ZSTD_error_checksum_wrong,
ZSTD_error_tableLog_tooLarge,
ZSTD_error_maxSymbolValue_tooLarge,
ZSTD_error_maxSymbolValue_tooSmall,
ZSTD_error_dictionary_corrupted,
ZSTD_error_dictionary_wrong,
ZSTD_error_maxCode
} ZSTD_ErrorCode;
/*! ZSTD_getErrorCode() :
convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
which can be used to compare directly with enum list published into "error_public.h" */
ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_ERRORS_H_398273423 */

6
C/zstd/zstd_opt.h
View File

@@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
{ U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
inr = ip;
//inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
ctx->nextToUpdate3 = ctx->nextToUpdate;
ZSTD_rescaleFreqs(seqStorePtr);
ip += (ip==prefixStart);
inr = ip;
//inr = ip;
/* Match Loop */
while (ip < ilimit) {
@@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
U32 current = (U32)(ip-base);
memset(opt, 0, sizeof(ZSTD_optimal_t));
last_pos = 0;
inr = ip;
//inr = ip;
opt[0].litlen = (U32)(ip - anchor);
/* check repCode */

5
C/zstd/zstd_v04.c
View File

@@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
}
}
/* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8)

5
C/zstd/zstd_v05.c
View File

@@ -3312,7 +3312,12 @@ static size_t ZSTDv05_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {

5
C/zstd/zstd_v06.c
View File

@@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_8) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_8 */
/* match within prefix */
if (sequence.offset < 8) {

5
C/zstd/zstd_v07.c
View File

@@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op,
op = oLitEnd + length1;
sequence.matchLength -= length1;
match = base;
if (op > oend_w) {
memmove(op, match, sequence.matchLength);
return sequenceLength;
}
} }
/* Requirement: op <= oend_w */
/* match within prefix */
if (sequence.offset < 8) {

66
C/zstdmt/README.md Normal file
View File

@@ -0,0 +1,66 @@
# Multithreading Library for [LZ4], [LZ5] and [ZStandard]
### Compression
```
typedef struct {
void *buf; /* ptr to data */
size_t size; /* current filled in buf */
size_t allocated; /* length of buf */
} LZ4MT_Buffer;
/**
* reading and writing functions
* - you can use stdio functions or plain read/write ...
* - a sample is given in 7-Zip ZS or lz4mt.c
*/
typedef int (fn_read) (void *args, LZ4MT_Buffer * in);
typedef int (fn_write) (void *args, LZ4MT_Buffer * out);
typedef struct {
fn_read *fn_read;
void *arg_read;
fn_write *fn_write;
void *arg_write;
} LZ4MT_RdWr_t;
typedef struct LZ4MT_CCtx_s LZ4MT_CCtx;
/* 1) allocate new cctx */
LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize);
/* 2) threaded compression */
size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr);
/* 3) get some statistic */
size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx);
size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx);
size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx);
/* 4) free cctx */
void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx);
```
### Decompression
```
typedef struct LZ4MT_DCtx_s LZ4MT_DCtx;
/* 1) allocate new cctx */
LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize);
/* 2) threaded compression */
size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr);
/* 3) get some statistic */
size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx);
size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx);
size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx);
/* 4) free cctx */
void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx);
```
## Todo
- add Makefile

130
C/zstdmt/list.h Normal file
View File

@@ -0,0 +1,130 @@
/**
* Copyright (c) 2016 Tino Reichardt
* - removed unneeded stuff
* - added win32 compatibility
*
* Copyright (c) 2013 The NetBSD Foundation, Inc.
* All rights reserved.
*
* This code is derived from software contributed to The NetBSD Foundation
* by Taylor R. Campbell.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef LIST_H
#define LIST_H
#if defined (__cplusplus)
extern "C" {
#endif
#include "mem.h"
struct list_head {
struct list_head *prev;
struct list_head *next;
};
MEM_STATIC void INIT_LIST_HEAD(struct list_head *head)
{
head->prev = head;
head->next = head;
}
MEM_STATIC struct list_head *list_first(const struct list_head *head)
{
return head->next;
}
MEM_STATIC struct list_head *list_last(const struct list_head *head)
{
return head->prev;
}
MEM_STATIC struct list_head *list_next(const struct list_head *node)
{
return node->next;
}
MEM_STATIC struct list_head *list_prev(const struct list_head *node)
{
return node->prev;
}
MEM_STATIC void __list_add_between(struct list_head *prev,
struct list_head *node, struct list_head *next)
{
prev->next = node;
node->prev = prev;
node->next = next;
next->prev = node;
}
MEM_STATIC void list_add(struct list_head *node, struct list_head *head)
{
__list_add_between(head, node, head->next);
}
MEM_STATIC void list_add_tail(struct list_head *node, struct list_head *head)
{
__list_add_between(head->prev, node, head);
}
MEM_STATIC void list_del(struct list_head *entry)
{
entry->prev->next = entry->next;
entry->next->prev = entry->prev;
}
MEM_STATIC int list_empty(const struct list_head *head)
{
return (head->next == head);
}
MEM_STATIC void list_move(struct list_head *node, struct list_head *head)
{
list_del(node);
list_add(node, head);
}
MEM_STATIC void list_move_tail(struct list_head *node, struct list_head *head)
{
list_del(node);
list_add_tail(node, head);
}
#ifndef CONTAINING_RECORD
#define CONTAINING_RECORD(ptr, type, field) \
((type*)((char*)(ptr) - (char*)(&((type*)0)->field)))
#endif
#define list_entry(ptr, type, member) CONTAINING_RECORD(ptr, type, member)
#define list_for_each(var, head) \
for ((var) = list_first((head)); (var) != (head); (var) = list_next((var)))
#if defined (__cplusplus)
}
#endif
#endif /* LIST_H */

162
C/zstdmt/lz4mt.h Normal file
View File

@@ -0,0 +1,162 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
/* ***************************************
* Defines
****************************************/
#ifndef LZ4MT_H
#define LZ4MT_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h> /* size_t */
/* current maximum the library will accept */
#define LZ4MT_THREAD_MAX 128
#define LZ4MT_LEVEL_MAX 16
#define LZ4FMT_MAGICNUMBER 0x184D2204U
#define LZ4FMT_MAGIC_SKIPPABLE 0x184D2A50U
/* **************************************
* Error Handling
****************************************/
extern size_t lz4mt_errcode;
typedef enum {
LZ4MT_error_no_error,
LZ4MT_error_memory_allocation,
LZ4MT_error_read_fail,
LZ4MT_error_write_fail,
LZ4MT_error_data_error,
LZ4MT_error_frame_compress,
LZ4MT_error_frame_decompress,
LZ4MT_error_compressionParameter_unsupported,
LZ4MT_error_compression_library,
LZ4MT_error_maxCode
} LZ4MT_ErrorCode;
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define PREFIX(name) LZ4MT_error_##name
#define ERROR(name) ((size_t)-PREFIX(name))
extern unsigned LZ4MT_isError(size_t code);
extern const char* LZ4MT_getErrorString(size_t code);
/* **************************************
* Structures
****************************************/
typedef struct {
void *buf; /* ptr to data */
size_t size; /* current filled in buf */
size_t allocated; /* length of buf */
} LZ4MT_Buffer;
/**
* reading and writing functions
* - you can use stdio functions or plain read/write
* - just write some wrapper on your own
* - a sample is given in 7-Zip ZS or lz4mt.c
* - the function should return -1 on error and zero on success
* - the read or written bytes will go to in->size or out->size
*/
typedef int (fn_read) (void *args, LZ4MT_Buffer * in);
typedef int (fn_write) (void *args, LZ4MT_Buffer * out);
typedef struct {
fn_read *fn_read;
void *arg_read;
fn_write *fn_write;
void *arg_write;
} LZ4MT_RdWr_t;
/* **************************************
* Compression
****************************************/
typedef struct LZ4MT_CCtx_s LZ4MT_CCtx;
/**
* 1) allocate new cctx
* - return cctx or zero on error
*
* @level - 1 .. 9
* @threads - 1 .. LZ4MT_THREAD_MAX
* @inputsize - if zero, becomes some optimal value for the level
* - if nonzero, the given value is taken
*/
LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize);
/**
* 2) threaded compression
* - errorcheck via
*/
size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr);
/**
* 3) get some statistic
*/
size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx);
size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx);
size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx);
/**
* 4) free cctx
* - no special return value
*/
void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx);
/* **************************************
* Decompression
****************************************/
typedef struct LZ4MT_DCtx_s LZ4MT_DCtx;
/**
* 1) allocate new cctx
* - return cctx or zero on error
*
* @threads - 1 .. LZ4MT_THREAD_MAX
* @ inputsize - used for single threaded standard lz4 format without skippable frames
*/
LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize);
/**
* 2) threaded compression
* - return -1 on error
*/
size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr);
/**
* 3) get some statistic
*/
size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx);
size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx);
size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx);
/**
* 4) free cctx
* - no special return value
*/
void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx);
#if defined (__cplusplus)
}
#endif
#endif /* LZ4MT_H */

62
C/zstdmt/lz4mt_common.c Normal file
View File

@@ -0,0 +1,62 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include "lz4frame.h"
#include "lz4mt.h"
/* will be used for lib errors */
size_t lz4mt_errcode;
/* ****************************************
* LZ4MT Error Management
******************************************/
/**
* LZ4MT_isError() - tells if a return value is an error code
*/
unsigned LZ4MT_isError(size_t code)
{
return (code > ERROR(maxCode));
}
/**
* LZ4MT_getErrorString() - give error code string from function result
*/
const char *LZ4MT_getErrorString(size_t code)
{
if (LZ4F_isError(lz4mt_errcode))
return LZ4F_getErrorName(lz4mt_errcode);
static const char *notErrorCode = "Unspecified error lz4mt code";
switch ((LZ4MT_ErrorCode)(0-code)) {
case PREFIX(no_error):
return "No error detected";
case PREFIX(memory_allocation):
return "Allocation error : not enough memory";
case PREFIX(read_fail):
return "Read failure";
case PREFIX(write_fail):
return "Write failure";
case PREFIX(data_error):
return "Malformed input";
case PREFIX(frame_compress):
return "Could not compress frame at once";
case PREFIX(frame_decompress):
return "Could not decompress frame at once";
case PREFIX(compressionParameter_unsupported):
return "Compression parameter is out of bound";
case PREFIX(compression_library):
return "Compression library reports failure";
case PREFIX(maxCode):
default:
return notErrorCode;
}
}

377
C/zstdmt/lz4mt_compress.c Normal file
View File

@@ -0,0 +1,377 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdlib.h>
#include <string.h>
#define LZ4F_DISABLE_OBSOLETE_ENUMS
#include "lz4frame.h"
#include "mem.h"
#include "threading.h"
#include "list.h"
#include "lz4mt.h"
/**
* multi threaded lz4 - multiple workers version
*
* - each thread works on his own
* - no main thread which does reading and then starting the work
* - needs a callback for reading / writing
* - each worker does his:
* 1) get read mutex and read some input
* 2) release read mutex and do compression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
/* worker for compression */
typedef struct {
LZ4MT_CCtx *ctx;
LZ4F_preferences_t zpref;
pthread_t pthread;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
LZ4MT_Buffer out;
struct list_head node;
};
struct LZ4MT_CCtx_s {
/* level: 1..22 */
int level;
/* threads: 1..LZ4MT_THREAD_MAX */
int threads;
/* should be used for read from input */
int inputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Compression
****************************************/
LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize)
{
LZ4MT_CCtx *ctx;
int t;
/* allocate ctx */
ctx = (LZ4MT_CCtx *) malloc(sizeof(LZ4MT_CCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > LZ4MT_THREAD_MAX)
return 0;
/* check level */
if (level < 1 || level > LZ4MT_LEVEL_MAX)
return 0;
/* calculate chunksize for one thread */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 64;
/* setup ctx */
ctx->level = level;
ctx->threads = threads;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
/* free -> busy -> out -> free -> ... */
INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */
INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */
INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_cwork;
for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
/* setup preferences for that thread */
memset(&w->zpref, 0, sizeof(LZ4F_preferences_t));
w->zpref.compressionLevel = level;
w->zpref.frameInfo.blockMode = LZ4F_blockLinked;
w->zpref.frameInfo.contentSize = 1;
w->zpref.frameInfo.contentChecksumFlag =
LZ4F_contentChecksumEnabled;
}
return ctx;
err_cwork:
free(ctx);
return 0;
}
/**
* pt_write - queue for compressed output
*/
static size_t pt_write(LZ4MT_CCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
/* the entry isn't the currently needed, return... */
if (wl->frame != ctx->curframe)
return 0;
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
int rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv == -1)
return ERROR(write_fail);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
static void *pt_compress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
LZ4MT_CCtx *ctx = w->ctx;
size_t result;
LZ4MT_Buffer in;
/* inbuf is constant */
in.size = ctx->inputsize;
in.buf = malloc(in.size);
if (!in.buf)
return (void *)ERROR(memory_allocation);
for (;;) {
struct list_head *entry;
struct writelist *wl;
int rv;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
wl->out.size =
LZ4F_compressFrameBound(ctx->inputsize,
&w->zpref) + 12;
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)ERROR(memory_allocation);
}
wl->out.size =
LZ4F_compressFrameBound(ctx->inputsize,
&w->zpref) + 12;;
wl->out.buf = malloc(wl->out.size);
if (!wl->out.buf) {
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)ERROR(memory_allocation);
}
list_add(&wl->node, &ctx->writelist_busy);
}
pthread_mutex_unlock(&ctx->write_mutex);
/* read new input */
pthread_mutex_lock(&ctx->read_mutex);
in.size = ctx->inputsize;
rv = ctx->fn_read(ctx->arg_read, &in);
if (rv == -1) {
pthread_mutex_unlock(&ctx->read_mutex);
return (void *)ERROR(read_fail);
}
/* eof */
if (in.size == 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
goto okay;
}
ctx->insize += in.size;
wl->frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* compress whole frame */
result =
LZ4F_compressFrame((unsigned char *)wl->out.buf + 12,
wl->out.size - 12, in.buf, in.size,
&w->zpref);
if (LZ4F_isError(result)) {
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
/* user can lookup that code */
lz4mt_errcode = result;
return (void *)ERROR(compression_library);
}
/* write skippable frame */
MEM_writeLE32((unsigned char *)wl->out.buf + 0,
LZ4FMT_MAGIC_SKIPPABLE);
MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4);
MEM_writeLE32((unsigned char *)wl->out.buf + 8,
(U32)result);
wl->out.size = result + 12;
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
pthread_mutex_unlock(&ctx->write_mutex);
if (LZ4MT_isError(result))
return (void *)result;
}
okay:
return 0;
}
size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr)
{
int t;
if (!ctx)
return ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* start all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
pthread_create(&w->pthread, NULL, pt_compress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
if (p)
return (size_t) p;
}
/* clean up lists */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
return 0;
}
/* returns current uncompressed data size */
size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx)
{
if (!ctx)
return 0;
return ctx->insize;
}
/* returns the current compressed data size */
size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx)
{
if (!ctx)
return 0;
return ctx->outsize;
}
/* returns the current compressed frames */
size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx)
{
if (!ctx)
return 0;
return ctx->curframe;
}
void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx)
{
if (!ctx)
return;
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

587
C/zstdmt/lz4mt_decompress.c Normal file
View File

@@ -0,0 +1,587 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdlib.h>
#include <string.h>
#define LZ4F_DISABLE_OBSOLETE_ENUMS
#include "lz4frame.h"
#include "mem.h"
#include "threading.h"
#include "list.h"
#include "lz4mt.h"
/**
* multi threaded lz4 - multiple workers version
*
* - each thread works on his own
* - no main thread which does reading and then starting the work
* - needs a callback for reading / writing
* - each worker does his:
* 1) get read mutex and read some input
* 2) release read mutex and do compression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
/* worker for compression */
typedef struct {
LZ4MT_DCtx *ctx;
pthread_t pthread;
LZ4MT_Buffer in;
LZ4F_decompressionContext_t dctx;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
LZ4MT_Buffer out;
struct list_head node;
};
struct LZ4MT_DCtx_s {
/* threads: 1..LZ4MT_THREAD_MAX */
int threads;
/* should be used for read from input */
size_t inputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Decompression
****************************************/
LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize)
{
LZ4MT_DCtx *ctx;
int t;
/* allocate ctx */
ctx = (LZ4MT_DCtx *) malloc(sizeof(LZ4MT_DCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > LZ4MT_THREAD_MAX)
return 0;
/* setup ctx */
ctx->threads = threads;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
/* will be used for single stream only */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 64; /* 64K buffer */
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_cwork;
for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
/* setup thread work */
LZ4F_createDecompressionContext(&w->dctx, LZ4F_VERSION);
}
return ctx;
err_cwork:
free(ctx);
return 0;
}
/**
* pt_write - queue for decompressed output
*/
static size_t pt_write(LZ4MT_DCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
int rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv == -1)
return ERROR(write_fail);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
/**
* pt_read - read compressed output
*/
static size_t pt_read(LZ4MT_DCtx * ctx, LZ4MT_Buffer * in, size_t * frame)
{
unsigned char hdrbuf[12];
LZ4MT_Buffer hdr;
int rv;
/* read skippable frame (8 or 12 bytes) */
pthread_mutex_lock(&ctx->read_mutex);
/* special case, first 4 bytes already read */
if (ctx->frames == 0) {
hdr.buf = hdrbuf + 4;
hdr.size = 8;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv == -1)
goto error_read;
if (hdr.size != 8)
goto error_read;
hdr.buf = hdrbuf;
} else {
hdr.buf = hdrbuf;
hdr.size = 12;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv == -1)
goto error_read;
/* eof reached ? */
if (hdr.size == 0) {
pthread_mutex_unlock(&ctx->read_mutex);
in->size = 0;
return 0;
}
if (hdr.size != 12)
goto error_read;
if (MEM_readLE32((unsigned char *)hdr.buf + 0) !=
LZ4FMT_MAGIC_SKIPPABLE)
goto error_data;
}
/* check header data */
if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4)
goto error_data;
ctx->insize += 12;
/* read new inputsize */
{
size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
if (in->allocated < toRead) {
/* need bigger input buffer */
if (in->allocated)
in->buf = realloc(in->buf, toRead);
else
in->buf = malloc(toRead);
if (!in->buf)
goto error_nomem;
in->allocated = toRead;
}
in->size = toRead;
rv = ctx->fn_read(ctx->arg_read, in);
/* generic read failure! */
if (rv == -1)
goto error_read;
/* needed more bytes! */
if (in->size != toRead)
goto error_data;
ctx->insize += in->size;
}
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* done, no error */
return 0;
error_data:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(data_error);
error_read:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(read_fail);
error_nomem:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(memory_allocation);
}
static void *pt_decompress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
LZ4MT_Buffer *in = &w->in;
LZ4MT_DCtx *ctx = w->ctx;
size_t result = 0;
struct writelist *wl;
for (;;) {
struct list_head *entry;
LZ4MT_Buffer *out;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
result = ERROR(memory_allocation);
goto error_unlock;
}
wl->out.buf = 0;
wl->out.size = 0;
wl->out.allocated = 0;
list_add(&wl->node, &ctx->writelist_busy);
}
pthread_mutex_unlock(&ctx->write_mutex);
out = &wl->out;
/* zero should not happen here! */
result = pt_read(ctx, in, &wl->frame);
if (LZ4MT_isError(result)) {
list_move(&wl->node, &ctx->writelist_free);
goto error_lock;
}
if (in->size == 0)
break;
{
/* get frame size for output buffer */
unsigned char *src = (unsigned char *)in->buf + 6;
out->size = (size_t) MEM_readLE64(src);
}
if (out->allocated < out->size) {
if (out->allocated)
out->buf = realloc(out->buf, out->size);
else
out->buf = malloc(out->size);
if (!out->buf) {
result = ERROR(memory_allocation);
goto error_lock;
}
out->allocated = out->size;
}
result =
LZ4F_decompress(w->dctx, out->buf, &out->size,
in->buf, &in->size, 0);
if (LZ4F_isError(result)) {
lz4mt_errcode = result;
result = ERROR(compression_library);
goto error_lock;
}
if (result != 0) {
result = ERROR(frame_decompress);
goto error_lock;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
if (LZ4MT_isError(result))
goto error_unlock;
pthread_mutex_unlock(&ctx->write_mutex);
}
/* everything is okay */
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return 0;
error_lock:
pthread_mutex_lock(&ctx->write_mutex);
error_unlock:
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return (void *)result;
}
/* single threaded */
static size_t st_decompress(void *arg)
{
LZ4MT_DCtx *ctx = (LZ4MT_DCtx *) arg;
LZ4F_errorCode_t nextToLoad = 0;
cwork_t *w = &ctx->cwork[0];
LZ4MT_Buffer Out;
LZ4MT_Buffer *out = &Out;
LZ4MT_Buffer *in = &w->in;
void *magic = in->buf;
size_t pos = 0;
int rv;
/* allocate space for input buffer */
in->size = ctx->inputsize;
in->buf = malloc(in->size);
if (!in->buf)
return ERROR(memory_allocation);
/* allocate space for output buffer */
out->size = ctx->inputsize;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return ERROR(memory_allocation);
}
/* we have read already 4 bytes */
in->size = 4;
memcpy(in->buf, magic, in->size);
nextToLoad =
LZ4F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
if (LZ4F_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
}
for (; nextToLoad; pos = 0) {
if (nextToLoad > ctx->inputsize)
nextToLoad = ctx->inputsize;
/* read new input */
in->size = nextToLoad;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1) {
free(in->buf);
free(out->buf);
return ERROR(read_fail);
}
/* done, eof reached */
if (in->size == 0)
break;
/* still to read, or still to flush */
while ((pos < in->size) || (out->size == ctx->inputsize)) {
size_t remaining = in->size - pos;
out->size = ctx->inputsize;
/* decompress */
nextToLoad =
LZ4F_decompress(w->dctx, out->buf, &out->size,
(unsigned char *)in->buf + pos,
&remaining, NULL);
if (LZ4F_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
}
/* have some output */
if (out->size) {
rv = ctx->fn_write(ctx->arg_write, out);
if (rv == -1) {
free(in->buf);
free(out->buf);
return ERROR(write_fail);
}
}
if (nextToLoad == 0)
break;
pos += remaining;
}
}
/* no error */
free(out->buf);
free(in->buf);
return 0;
}
size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr)
{
unsigned char buf[4];
int t, rv;
cwork_t *w = &ctx->cwork[0];
LZ4MT_Buffer *in = &w->in;
if (!ctx)
return ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* check for LZ4FMT_MAGIC_SKIPPABLE */
in->buf = buf;
in->size = 4;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1)
return ERROR(read_fail);
if (in->size != 4)
return ERROR(data_error);
/* single threaded with unknown sizes */
if (MEM_readLE32(buf) != LZ4FMT_MAGIC_SKIPPABLE) {
/* look for correct magic */
if (MEM_readLE32(buf) != LZ4FMT_MAGICNUMBER)
return ERROR(data_error);
/* decompress single threaded */
return st_decompress(ctx);
}
/* mark unused */
in->buf = 0;
in->size = 0;
in->allocated = 0;
/* single threaded, but with known sizes */
if (ctx->threads == 1) {
/* no pthread_create() needed! */
void *p = pt_decompress(w);
if (p)
return (size_t) p;
goto okay;
}
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->in.buf = 0;
w->in.size = 0;
w->in.allocated = 0;
pthread_create(&w->pthread, NULL, pt_decompress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
if (p)
return (size_t) p;
}
okay:
/* clean up the buffers */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
return 0;
}
/* returns current uncompressed data size */
size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->insize;
}
/* returns the current compressed data size */
size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->outsize;
}
/* returns the current compressed frames */
size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->curframe;
}
void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx)
{
int t;
if (!ctx)
return;
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
LZ4F_freeDecompressionContext(w->dctx);
}
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

162
C/zstdmt/lz5mt.h Normal file
View File

@@ -0,0 +1,162 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
/* ***************************************
* Defines
****************************************/
#ifndef LZ5MT_H
#define LZ5MT_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h> /* size_t */
/* current maximum the library will accept */
#define LZ5MT_THREAD_MAX 128
#define LZ5MT_LEVEL_MAX 16
#define LZ5FMT_MAGICNUMBER 0x184D2205U
#define LZ5FMT_MAGIC_SKIPPABLE 0x184D2A50U
/* **************************************
* Error Handling
****************************************/
extern size_t lz5mt_errcode;
typedef enum {
LZ5MT_error_no_error,
LZ5MT_error_memory_allocation,
LZ5MT_error_read_fail,
LZ5MT_error_write_fail,
LZ5MT_error_data_error,
LZ5MT_error_frame_compress,
LZ5MT_error_frame_decompress,
LZ5MT_error_compressionParameter_unsupported,
LZ5MT_error_compression_library,
LZ5MT_error_maxCode
} LZ5MT_ErrorCode;
#ifdef ERROR
# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */
#endif
#define PREFIX(name) LZ5MT_error_##name
#define ERROR(name) ((size_t)-PREFIX(name))
extern unsigned LZ5MT_isError(size_t code);
extern const char* LZ5MT_getErrorString(size_t code);
/* **************************************
* Structures
****************************************/
typedef struct {
void *buf; /* ptr to data */
size_t size; /* current filled in buf */
size_t allocated; /* length of buf */
} LZ5MT_Buffer;
/**
* reading and writing functions
* - you can use stdio functions or plain read/write
* - just write some wrapper on your own
* - a sample is given in 7-Zip ZS
*/
typedef int (fn_read) (void *args, LZ5MT_Buffer * in);
typedef int (fn_write) (void *args, LZ5MT_Buffer * out);
typedef struct {
fn_read *fn_read;
void *arg_read;
fn_write *fn_write;
void *arg_write;
} LZ5MT_RdWr_t;
/* **************************************
* Compression
****************************************/
typedef struct LZ5MT_CCtx_s LZ5MT_CCtx;
/**
* 1) allocate new cctx
* - return cctx or zero on error
*
* @level - 1 .. 16
* @threads - 1 .. LZ5MT_THREAD_MAX
* @inputsize - if zero, becomes some optimal value for the level
* - if nonzero, the given value is taken
*/
LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize);
/**
* 2) threaded compression
* - return -1 on error
*/
size_t LZ5MT_CompressCCtx(LZ5MT_CCtx * ctx, LZ5MT_RdWr_t * rdwr);
/**
* 3) get some statistic
*/
size_t LZ5MT_GetFramesCCtx(LZ5MT_CCtx * ctx);
size_t LZ5MT_GetInsizeCCtx(LZ5MT_CCtx * ctx);
size_t LZ5MT_GetOutsizeCCtx(LZ5MT_CCtx * ctx);
/**
* 4) free cctx
* - no special return value
*/
void LZ5MT_freeCCtx(LZ5MT_CCtx * ctx);
/* **************************************
* Decompression - TODO, but it's easy...
****************************************/
typedef struct LZ5MT_DCtx_s LZ5MT_DCtx;
/**
* 1) allocate new cctx
* - return cctx or zero on error
*
* @level - 1 .. 22
* @threads - 1 .. LZ5MT_THREAD_MAX
* @srclen - the max size of src for LZ5MT_CompressCCtx()
* @dstlen - the min size of dst
*/
LZ5MT_DCtx *LZ5MT_createDCtx(int threads, int inputsize);
/**
* 2) threaded compression
* - return -1 on error
*/
size_t LZ5MT_DecompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr);
/**
* 3) get some statistic
*/
size_t LZ5MT_GetFramesDCtx(LZ5MT_DCtx * ctx);
size_t LZ5MT_GetInsizeDCtx(LZ5MT_DCtx * ctx);
size_t LZ5MT_GetOutsizeDCtx(LZ5MT_DCtx * ctx);
/**
* 4) free cctx
* - no special return value
*/
void LZ5MT_freeDCtx(LZ5MT_DCtx * ctx);
#if defined (__cplusplus)
}
#endif
#endif /* LZ5MT_H */

62
C/zstdmt/lz5mt_common.c Normal file
View File

@@ -0,0 +1,62 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include "lz5frame.h"
#include "lz5mt.h"
/* will be used for lib errors */
size_t lz5mt_errcode;
/* ****************************************
* LZ5MT Error Management
******************************************/
/**
* LZ5MT_isError() - tells if a return value is an error code
*/
unsigned LZ5MT_isError(size_t code)
{
return (code > ERROR(maxCode));
}
/**
* LZ5MT_getErrorString() - give error code string from function result
*/
const char *LZ5MT_getErrorString(size_t code)
{
if (LZ5F_isError(lz5mt_errcode))
return LZ5F_getErrorName(lz5mt_errcode);
static const char *notErrorCode = "Unspecified error lz4mt code";
switch ((LZ5MT_ErrorCode)(0-code)) {
case PREFIX(no_error):
return "No error detected";
case PREFIX(memory_allocation):
return "Allocation error : not enough memory";
case PREFIX(read_fail):
return "Read failure";
case PREFIX(write_fail):
return "Write failure";
case PREFIX(data_error):
return "Malformed input";
case PREFIX(frame_compress):
return "Could not compress frame at once";
case PREFIX(frame_decompress):
return "Could not decompress frame at once";
case PREFIX(compressionParameter_unsupported):
return "Compression parameter is out of bound";
case PREFIX(compression_library):
return "Compression library reports failure";
case PREFIX(maxCode):
default:
return notErrorCode;
}
}

377
C/zstdmt/lz5mt_compress.c Normal file
View File

@@ -0,0 +1,377 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdlib.h>
#include <string.h>
#define LZ5F_DISABLE_OBSOLETE_ENUMS
#include "lz5frame.h"
#include "mem.h"
#include "threading.h"
#include "list.h"
#include "lz5mt.h"
/**
* multi threaded lz5 - multiple workers version
*
* - each thread works on his own
* - no main thread which does reading and then starting the work
* - needs a callback for reading / writing
* - each worker does his:
* 1) get read mutex and read some input
* 2) release read mutex and do compression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
/* worker for compression */
typedef struct {
LZ5MT_CCtx *ctx;
LZ5F_preferences_t zpref;
pthread_t pthread;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
LZ5MT_Buffer out;
struct list_head node;
};
struct LZ5MT_CCtx_s {
/* level: 1..22 */
int level;
/* threads: 1..LZ5MT_THREAD_MAX */
int threads;
/* should be used for read from input */
int inputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Compression
****************************************/
LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize)
{
LZ5MT_CCtx *ctx;
int t;
/* allocate ctx */
ctx = (LZ5MT_CCtx *) malloc(sizeof(LZ5MT_CCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > LZ5MT_THREAD_MAX)
return 0;
/* check level */
if (level < 1 || level > LZ5MT_LEVEL_MAX)
return 0;
/* calculate chunksize for one thread */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 64;
/* setup ctx */
ctx->level = level;
ctx->threads = threads;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
/* free -> busy -> out -> free -> ... */
INIT_LIST_HEAD(&ctx->writelist_free); /* free, can be used */
INIT_LIST_HEAD(&ctx->writelist_busy); /* busy */
INIT_LIST_HEAD(&ctx->writelist_done); /* can be written */
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_cwork;
for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
/* setup preferences for that thread */
memset(&w->zpref, 0, sizeof(LZ5F_preferences_t));
w->zpref.compressionLevel = level;
w->zpref.frameInfo.blockMode = LZ5F_blockLinked;
w->zpref.frameInfo.contentSize = 1;
w->zpref.frameInfo.contentChecksumFlag =
LZ5F_contentChecksumEnabled;
}
return ctx;
err_cwork:
free(ctx);
return 0;
}
/**
* pt_write - queue for compressed output
*/
static size_t pt_write(LZ5MT_CCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
/* the entry isn't the currently needed, return... */
if (wl->frame != ctx->curframe)
return 0;
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
int rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv == -1)
return ERROR(write_fail);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
static void *pt_compress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
LZ5MT_CCtx *ctx = w->ctx;
size_t result;
LZ5MT_Buffer in;
/* inbuf is constant */
in.size = ctx->inputsize;
in.buf = malloc(in.size);
if (!in.buf)
return (void *)ERROR(memory_allocation);
for (;;) {
struct list_head *entry;
struct writelist *wl;
int rv;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
wl->out.size =
LZ5F_compressFrameBound(ctx->inputsize,
&w->zpref) + 12;
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)ERROR(memory_allocation);
}
wl->out.size =
LZ5F_compressFrameBound(ctx->inputsize,
&w->zpref) + 12;;
wl->out.buf = malloc(wl->out.size);
if (!wl->out.buf) {
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)ERROR(memory_allocation);
}
list_add(&wl->node, &ctx->writelist_busy);
}
pthread_mutex_unlock(&ctx->write_mutex);
/* read new input */
pthread_mutex_lock(&ctx->read_mutex);
in.size = ctx->inputsize;
rv = ctx->fn_read(ctx->arg_read, &in);
if (rv == -1) {
pthread_mutex_unlock(&ctx->read_mutex);
return (void *)ERROR(read_fail);
}
/* eof */
if (in.size == 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
goto okay;
}
ctx->insize += in.size;
wl->frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* compress whole frame */
result =
LZ5F_compressFrame((unsigned char *)wl->out.buf + 12,
wl->out.size - 12, in.buf, in.size,
&w->zpref);
if (LZ5F_isError(result)) {
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
/* user can lookup that code */
lz5mt_errcode = result;
return (void *)ERROR(compression_library);
}
/* write skippable frame */
MEM_writeLE32((unsigned char *)wl->out.buf + 0,
LZ5FMT_MAGIC_SKIPPABLE);
MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4);
MEM_writeLE32((unsigned char *)wl->out.buf + 8,
(U32)result);
wl->out.size = result + 12;
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
pthread_mutex_unlock(&ctx->write_mutex);
if (LZ5MT_isError(result))
return (void *)result;
}
okay:
return 0;
}
size_t LZ5MT_CompressCCtx(LZ5MT_CCtx * ctx, LZ5MT_RdWr_t * rdwr)
{
int t;
if (!ctx)
return ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* start all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
pthread_create(&w->pthread, NULL, pt_compress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
if (p)
return (size_t) p;
}
/* clean up lists */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
return 0;
}
/* returns current uncompressed data size */
size_t LZ5MT_GetInsizeCCtx(LZ5MT_CCtx * ctx)
{
if (!ctx)
return 0;
return ctx->insize;
}
/* returns the current compressed data size */
size_t LZ5MT_GetOutsizeCCtx(LZ5MT_CCtx * ctx)
{
if (!ctx)
return 0;
return ctx->outsize;
}
/* returns the current compressed frames */
size_t LZ5MT_GetFramesCCtx(LZ5MT_CCtx * ctx)
{
if (!ctx)
return 0;
return ctx->curframe;
}
void LZ5MT_freeCCtx(LZ5MT_CCtx * ctx)
{
if (!ctx)
return;
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

588
C/zstdmt/lz5mt_decompress.c Normal file
View File

@@ -0,0 +1,588 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdlib.h>
#include <string.h>
#define LZ5F_DISABLE_OBSOLETE_ENUMS
#include "lz5frame.h"
#include "mem.h"
#include "threading.h"
#include "list.h"
#include "lz5mt.h"
/**
* multi threaded lz5 - multiple workers version
*
* - each thread works on his own
* - no main thread which does reading and then starting the work
* - needs a callback for reading / writing
* - each worker does his:
* 1) get read mutex and read some input
* 2) release read mutex and do compression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
/* worker for compression */
typedef struct {
LZ5MT_DCtx *ctx;
pthread_t pthread;
LZ5MT_Buffer in;
LZ5F_decompressionContext_t dctx;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
LZ5MT_Buffer out;
struct list_head node;
};
struct LZ5MT_DCtx_s {
/* threads: 1..LZ5MT_THREAD_MAX */
int threads;
/* should be used for read from input */
size_t inputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Decompression
****************************************/
LZ5MT_DCtx *LZ5MT_createDCtx(int threads, int inputsize)
{
LZ5MT_DCtx *ctx;
int t;
/* allocate ctx */
ctx = (LZ5MT_DCtx *) malloc(sizeof(LZ5MT_DCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > LZ5MT_THREAD_MAX)
return 0;
/* setup ctx */
ctx->threads = threads;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
/* will be used for single stream only */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 64; /* 64K buffer */
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_cwork;
for (t = 0; t < threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
/* setup thread work */
LZ5F_createDecompressionContext(&w->dctx, LZ5F_VERSION);
}
return ctx;
err_cwork:
free(ctx);
return 0;
}
/**
* pt_write - queue for decompressed output
*/
static size_t pt_write(LZ5MT_DCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
int rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv == -1)
return ERROR(write_fail);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
/**
* pt_read - read compressed output
*/
static size_t pt_read(LZ5MT_DCtx * ctx, LZ5MT_Buffer * in, size_t * frame)
{
unsigned char hdrbuf[12];
LZ5MT_Buffer hdr;
int rv;
/* read skippable frame (8 or 12 bytes) */
pthread_mutex_lock(&ctx->read_mutex);
/* special case, first 4 bytes already read */
if (ctx->frames == 0) {
hdr.buf = hdrbuf + 4;
hdr.size = 8;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv == -1)
goto error_read;
if (hdr.size != 8)
goto error_read;
hdr.buf = hdrbuf;
} else {
hdr.buf = hdrbuf;
hdr.size = 12;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv == -1)
goto error_read;
/* eof reached ? */
if (hdr.size == 0) {
pthread_mutex_unlock(&ctx->read_mutex);
in->size = 0;
return 0;
}
if (hdr.size != 12)
goto error_read;
if (MEM_readLE32((unsigned char *)hdr.buf + 0) !=
LZ5FMT_MAGIC_SKIPPABLE)
goto error_data;
}
/* check header data */
if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4)
goto error_data;
ctx->insize += 12;
/* read new inputsize */
{
size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
if (in->allocated < toRead) {
/* need bigger input buffer */
if (in->allocated)
in->buf = realloc(in->buf, toRead);
else
in->buf = malloc(toRead);
if (!in->buf)
goto error_nomem;
in->allocated = toRead;
}
in->size = toRead;
rv = ctx->fn_read(ctx->arg_read, in);
/* generic read failure! */
if (rv == -1)
goto error_read;
/* needed more bytes! */
if (in->size != toRead)
goto error_data;
ctx->insize += in->size;
}
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* done, no error */
return 0;
error_data:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(data_error);
error_read:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(read_fail);
error_nomem:
pthread_mutex_unlock(&ctx->read_mutex);
return ERROR(memory_allocation);
}
static void *pt_decompress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
LZ5MT_Buffer *in = &w->in;
LZ5MT_DCtx *ctx = w->ctx;
size_t result = 0;
struct writelist *wl;
for (;;) {
struct list_head *entry;
LZ5MT_Buffer *out;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
result = ERROR(memory_allocation);
goto error_unlock;
}
wl->out.buf = 0;
wl->out.size = 0;
wl->out.allocated = 0;
list_add(&wl->node, &ctx->writelist_busy);
}
pthread_mutex_unlock(&ctx->write_mutex);
out = &wl->out;
/* zero should not happen here! */
result = pt_read(ctx, in, &wl->frame);
if (in->size == 0)
break;
if (LZ5MT_isError(result)) {
list_move(&wl->node, &ctx->writelist_free);
goto error_lock;
}
{
/* get frame size for output buffer */
unsigned char *src = (unsigned char *)in->buf + 6;
out->size = (size_t) MEM_readLE64(src);
}
if (out->allocated < out->size) {
if (out->allocated)
out->buf = realloc(out->buf, out->size);
else
out->buf = malloc(out->size);
if (!out->buf) {
result = ERROR(memory_allocation);
goto error_lock;
}
out->allocated = out->size;
}
result =
LZ5F_decompress(w->dctx, out->buf, &out->size,
in->buf, &in->size, 0);
if (LZ5F_isError(result)) {
lz5mt_errcode = result;
result = ERROR(compression_library);
goto error_lock;
}
if (result != 0) {
result = ERROR(frame_decompress);
goto error_lock;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
if (LZ5MT_isError(result))
goto error_unlock;
pthread_mutex_unlock(&ctx->write_mutex);
}
/* everything is okay */
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return 0;
error_lock:
pthread_mutex_lock(&ctx->write_mutex);
error_unlock:
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return (void *)result;
}
/* single threaded */
static size_t st_decompress(void *arg)
{
LZ5MT_DCtx *ctx = (LZ5MT_DCtx *) arg;
LZ5F_errorCode_t nextToLoad = 0;
cwork_t *w = &ctx->cwork[0];
LZ5MT_Buffer Out;
LZ5MT_Buffer *out = &Out;
LZ5MT_Buffer *in = &w->in;
void *magic = in->buf;
size_t pos = 0;
int rv;
/* allocate space for input buffer */
in->size = ctx->inputsize;
in->buf = malloc(in->size);
if (!in->buf)
return ERROR(memory_allocation);
/* allocate space for output buffer */
out->size = ctx->inputsize;
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return ERROR(memory_allocation);
}
/* we have read already 4 bytes */
in->size = 4;
memcpy(in->buf, magic, in->size);
nextToLoad =
LZ5F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
if (LZ5F_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
}
for (; nextToLoad; pos = 0) {
if (nextToLoad > ctx->inputsize)
nextToLoad = ctx->inputsize;
/* read new input */
in->size = nextToLoad;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1) {
free(in->buf);
free(out->buf);
return ERROR(read_fail);
}
/* done, eof reached */
if (in->size == 0)
break;
/* still to read, or still to flush */
while ((pos < in->size) || (out->size == ctx->inputsize)) {
size_t remaining = in->size - pos;
out->size = ctx->inputsize;
/* decompress */
nextToLoad =
LZ5F_decompress(w->dctx, out->buf, &out->size,
(unsigned char *)in->buf + pos,
&remaining, NULL);
if (LZ5F_isError(nextToLoad)) {
free(in->buf);
free(out->buf);
return ERROR(compression_library);
}
/* have some output */
if (out->size) {
rv = ctx->fn_write(ctx->arg_write, out);
if (rv == -1) {
free(in->buf);
free(out->buf);
return ERROR(write_fail);
}
}
if (nextToLoad == 0)
break;
pos += remaining;
}
}
/* no error */
free(out->buf);
free(in->buf);
return 0;
}
size_t LZ5MT_DecompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr)
{
unsigned char buf[4];
int t, rv;
cwork_t *w = &ctx->cwork[0];
LZ5MT_Buffer *in = &w->in;
if (!ctx)
return ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* check for LZ5FMT_MAGIC_SKIPPABLE */
in->buf = buf;
in->size = 4;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1)
return ERROR(read_fail);
if (in->size != 4)
return ERROR(data_error);
/* single threaded with unknown sizes */
if (MEM_readLE32(buf) != LZ5FMT_MAGIC_SKIPPABLE) {
/* look for correct magic */
if (MEM_readLE32(buf) != LZ5FMT_MAGICNUMBER)
return ERROR(data_error);
/* decompress single threaded */
return st_decompress(ctx);
}
/* mark unused */
in->buf = 0;
in->size = 0;
in->allocated = 0;
/* single threaded, but with known sizes */
if (ctx->threads == 1) {
/* no pthread_create() needed! */
void *p = pt_decompress(w);
if (p)
return (size_t) p;
goto okay;
}
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->in.buf = 0;
w->in.size = 0;
w->in.allocated = 0;
pthread_create(&w->pthread, NULL, pt_decompress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
if (p)
return (size_t) p;
}
okay:
/* clean up the buffers */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
return 0;
}
/* returns current uncompressed data size */
size_t LZ5MT_GetInsizeDCtx(LZ5MT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->insize;
}
/* returns the current compressed data size */
size_t LZ5MT_GetOutsizeDCtx(LZ5MT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->outsize;
}
/* returns the current compressed frames */
size_t LZ5MT_GetFramesDCtx(LZ5MT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->curframe;
}
void LZ5MT_freeDCtx(LZ5MT_DCtx * ctx)
{
int t;
if (!ctx)
return;
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
LZ5F_freeDecompressionContext(w->dctx);
}
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

379
C/zstdmt/mem.h Normal file
View File

@@ -0,0 +1,379 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */
/*-****************************************
* Compiler specifics
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined(__GNUC__)
# define MEM_STATIC static __inline __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
# define MEM_STATIC static __inline
#else
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
/* define likely() and unlikely() */
#if defined(__GNUC__) || (__INTEL_COMPILER >= 800) || defined(__clang__)
# define expect(expr,value) (__builtin_expect ((expr),(value)) )
#else
# define expect(expr,value) (expr)
#endif
#define likely(expr) expect((expr) != 0, 1)
#define unlikely(expr) expect((expr) != 0, 0)
/*-**************************************************************
* Basic Types
*****************************************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef int16_t S16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef signed short S16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
#endif
/*-**************************************************************
* Memory I/O
*****************************************************************/
/* MEM_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets depending on alignment.
* In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
(defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
#endif
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
MEM_STATIC unsigned MEM_isLittleEndian(void)
{
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
}
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
/* violates C standard, by lying on structure alignment.
Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
__pragma( pack(push, 1) )
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
__pragma( pack(pop) )
#else
typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
#endif
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
#else
/* default method, safe and standard.
can sometimes prove slower */
MEM_STATIC U16 MEM_read16(const void* memPtr)
{
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U32 MEM_read32(const void* memPtr)
{
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U64 MEM_read64(const void* memPtr)
{
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC size_t MEM_readST(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif /* MEM_FORCE_MEMORY_ACCESS */
MEM_STATIC U32 MEM_swap32(U32 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_ulong(in);
#elif defined (__GNUC__)
return __builtin_bswap32(in);
#else
return ((in << 24) & 0xff000000 ) |
((in << 8) & 0x00ff0000 ) |
((in >> 8) & 0x0000ff00 ) |
((in >> 24) & 0x000000ff );
#endif
}
MEM_STATIC U64 MEM_swap64(U64 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_uint64(in);
#elif defined (__GNUC__)
return __builtin_bswap64(in);
#else
return ((in << 56) & 0xff00000000000000ULL) |
((in << 40) & 0x00ff000000000000ULL) |
((in << 24) & 0x0000ff0000000000ULL) |
((in << 8) & 0x000000ff00000000ULL) |
((in >> 8) & 0x00000000ff000000ULL) |
((in >> 24) & 0x0000000000ff0000ULL) |
((in >> 40) & 0x000000000000ff00ULL) |
((in >> 56) & 0x00000000000000ffULL);
#endif
}
MEM_STATIC size_t MEM_swapST(size_t in)
{
if (MEM_32bits())
return (size_t)MEM_swap32((U32)in);
else
return (size_t)MEM_swap64((U64)in);
}
/*=== Little endian r/w ===*/
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read16(memPtr);
else {
const BYTE* p = (const BYTE*)memPtr;
return (U16)(p[0] + (p[1]<<8));
}
}
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
{
if (MEM_isLittleEndian()) {
MEM_write16(memPtr, val);
} else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val;
p[1] = (BYTE)(val>>8);
}
}
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
{
MEM_writeLE16(memPtr, (U16)val);
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
}
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read32(memPtr);
else
return MEM_swap32(MEM_read32(memPtr));
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, val32);
else
MEM_write32(memPtr, MEM_swap32(val32));
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read64(memPtr);
else
return MEM_swap64(MEM_read64(memPtr));
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, val64);
else
MEM_write64(memPtr, MEM_swap64(val64));
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readLE32(memPtr);
else
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
/*=== Big endian r/w ===*/
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap32(MEM_read32(memPtr));
else
return MEM_read32(memPtr);
}
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, MEM_swap32(val32));
else
MEM_write32(memPtr, val32);
}
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap64(MEM_read64(memPtr));
else
return MEM_read64(memPtr);
}
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, MEM_swap64(val64));
else
MEM_write64(memPtr, val64);
}
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readBE32(memPtr);
else
return (size_t)MEM_readBE64(memPtr);
}
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeBE32(memPtr, (U32)val);
else
MEM_writeBE64(memPtr, (U64)val);
}
/* function safe only for comparisons */
MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
{
switch (length)
{
default :
case 4 : return MEM_read32(memPtr);
case 3 : if (MEM_isLittleEndian())
return MEM_read32(memPtr)<<8;
else
return MEM_read32(memPtr)>>8;
}
}
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */

71
C/zstdmt/threading.c Normal file
View File

@@ -0,0 +1,71 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
/**
* This file will hold wrapper for systems, which do not support Pthreads
*/
#ifdef _WIN32
/**
* Windows Pthread Wrapper, based on this site:
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
*/
#include "threading.h"
#include <process.h>
#include <errno.h>
static unsigned __stdcall worker(void *arg)
{
pthread_t *thread = (pthread_t *) arg;
thread->arg = thread->start_routine(thread->arg);
return 0;
}
int
pthread_create(pthread_t * thread, const void *unused,
void *(*start_routine) (void *), void *arg)
{
(void)unused;
thread->arg = arg;
thread->start_routine = start_routine;
thread->handle =
(HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
if (!thread->handle)
return errno;
else
return 0;
}
int _pthread_join(pthread_t * thread, void **value_ptr)
{
if (!thread->handle)
return 0;
DWORD result = WaitForSingleObject(thread->handle, INFINITE);
switch (result) {
case WAIT_OBJECT_0:
if (value_ptr)
*value_ptr = thread->arg;
return 0;
case WAIT_ABANDONED:
return EINVAL;
default:
return GetLastError();
}
}
#endif

68
C/zstdmt/threading.h Normal file
View File

@@ -0,0 +1,68 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#ifndef THREADING_H
#define THREADING_H
#if defined (__cplusplus)
extern "C" {
#endif
#ifdef _WIN32
/**
* Windows Pthread Wrapper, based on this site:
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
*/
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
/* mutex */
#define pthread_mutex_t CRITICAL_SECTION
#define pthread_mutex_init(a,b) InitializeCriticalSection((a))
#define pthread_mutex_destroy(a) DeleteCriticalSection((a))
#define pthread_mutex_lock EnterCriticalSection
#define pthread_mutex_unlock LeaveCriticalSection
/* pthread_create() and pthread_join() */
typedef struct {
HANDLE handle;
void *(*start_routine) (void *);
void *arg;
} pthread_t;
extern int pthread_create(pthread_t * thread, const void *unused,
void *(*start_routine) (void *), void *arg);
#define pthread_join(a, b) _pthread_join(&(a), (b))
extern int _pthread_join(pthread_t * thread, void **value_ptr);
/**
* add here more systems as required
*/
#else
/* POSIX Systems */
#include <pthread.h>
#endif /* POSIX Systems */
#if defined (__cplusplus)
}
#endif
#endif /* PTHREAD_H */

202
C/zstdmt/zstdmt.h Normal file
View File

@@ -0,0 +1,202 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
/* ***************************************
* Defines
****************************************/
#ifndef ZSTDMT_H
#define ZSTDMT_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h> /* size_t */
#define ZSTDMT_THREAD_MAX 128
#define ZSTDMT_LEVEL_MAX 22
/* zstd magic values */
#define ZSTDMT_MAGICNUMBER_V01 0x1EB52FFDU
#define ZSTDMT_MAGICNUMBER_MIN 0xFD2FB522U
#define ZSTDMT_MAGICNUMBER_MAX 0xFD2FB528U
#define ZSTDMT_MAGIC_SKIPPABLE 0x184D2A50U
/* **************************************
* Error Handling
****************************************/
typedef enum {
ZSTDMT_error_no_error,
ZSTDMT_error_memory_allocation,
ZSTDMT_error_init_missing,
ZSTDMT_error_read_fail,
ZSTDMT_error_write_fail,
ZSTDMT_error_data_error,
ZSTDMT_error_frame_compress,
ZSTDMT_error_frame_decompress,
ZSTDMT_error_compressionParameter_unsupported,
ZSTDMT_error_compression_library,
ZSTDMT_error_maxCode
} ZSTDMT_ErrorCode;
extern size_t zstdmt_errcode;
#define ZSTDMT_PREFIX(name) ZSTDMT_error_##name
#define ZSTDMT_ERROR(name) ((size_t)-ZSTDMT_PREFIX(name))
extern unsigned ZSTDMT_isError(size_t code);
extern const char* ZSTDMT_getErrorString(size_t code);
/* **************************************
* Structures
****************************************/
typedef struct {
void *buf; /* ptr to data */
size_t size; /* current filled in buf */
size_t allocated; /* length of buf */
} ZSTDMT_Buffer;
/**
* reading and writing functions
* - you can use stdio functions or plain read/write
* - just write some wrapper on your own
* - a sample is given in 7-Zip ZS
*/
typedef int (fn_read) (void *args, ZSTDMT_Buffer * in);
typedef int (fn_write) (void *args, ZSTDMT_Buffer * out);
typedef struct {
fn_read *fn_read;
void *arg_read;
fn_write *fn_write;
void *arg_write;
} ZSTDMT_RdWr_t;
/* **************************************
* Compression
****************************************/
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
/**
* ZSTDMT_createCCtx() - allocate new compression context
*
* This function allocates and initializes an zstd commpression context.
* The context can be used multiple times without the need for resetting
* or re-initializing.
*
* @level: compression level, which should be used (1..22)
* @threads: number of threads, which should be used (1..ZSTDMT_THREAD_MAX)
* @inputsize: - if zero, becomes some optimal value for the level
* - if nonzero, the given value is taken
* @zstdmt_errcode: space for storing zstd errors (needed for thread safety)
* @return: the context on success, zero on error
*/
ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize);
/**
* ZSTDMT_compressDCtx() - threaded compression for zstd
*
* This function will create valid zstd streams. The number of threads,
* the input chunksize and the compression level are ....
*
* @ctx: context, which needs to be created with ZSTDMT_createDCtx()
* @rdwr: callback structure, which defines reding/writing functions
* @return: zero on success, or error code
*/
size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx * ctx, ZSTDMT_RdWr_t * rdwr);
/**
* ZSTDMT_GetFramesCCtx() - number of written frames
* ZSTDMT_GetInsizeCCtx() - read bytes of input
* ZSTDMT_GetOutsizeCCtx() - written bytes of output
*
* These three functions will return some statistical data of the
* compression context ctx.
*
* @ctx: context, which should be examined
* @return: the request value, or zero on error
*/
size_t ZSTDMT_GetFramesCCtx(ZSTDMT_CCtx * ctx);
size_t ZSTDMT_GetInsizeCCtx(ZSTDMT_CCtx * ctx);
size_t ZSTDMT_GetOutsizeCCtx(ZSTDMT_CCtx * ctx);
/**
* ZSTDMT_freeCCtx() - free compression context
*
* This function will free all allocated resources, which were allocated
* by ZSTDMT_createCCtx(). This function can not fail.
*
* @ctx: context, which should be freed
*/
void ZSTDMT_freeCCtx(ZSTDMT_CCtx * ctx);
/* **************************************
* Decompression
****************************************/
typedef struct ZSTDMT_DCtx_s ZSTDMT_DCtx;
/**
* 1) allocate new cctx
* - return cctx or zero on error
*
* @level - 1 .. 22
* @threads - 1 .. ZSTDMT_THREAD_MAX
* @srclen - the max size of src for ZSTDMT_compressCCtx()
* @dstlen - the min size of dst
*/
ZSTDMT_DCtx *ZSTDMT_createDCtx(int threads, int inputsize);
/**
* ZSTDMT_decompressDCtx() - threaded decompression for zstd
*
* This function will decompress valid zstd streams.
*
* @ctx: context, which needs to be created with ZSTDMT_createDCtx()
* @rdwr: callback structure, which defines reding/writing functions
* @return: zero on success, or error code
*/
size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr);
/**
* ZSTDMT_GetFramesDCtx() - number of read frames
* ZSTDMT_GetInsizeDCtx() - read bytes of input
* ZSTDMT_GetOutsizeDCtx() - written bytes of output
*
* These three functions will return some statistical data of the
* decompression context ctx.
*
* @ctx: context, which should be examined
* @return: the request value, or zero on error
*/
size_t ZSTDMT_GetFramesDCtx(ZSTDMT_DCtx * ctx);
size_t ZSTDMT_GetInsizeDCtx(ZSTDMT_DCtx * ctx);
size_t ZSTDMT_GetOutsizeDCtx(ZSTDMT_DCtx * ctx);
/**
* ZSTDMT_freeDCtx() - free decompression context
*
* This function will free all allocated resources, which were allocated
* by ZSTDMT_createDCtx(). This function can not fail.
*
* @ctx: context, which should be freed
*/
void ZSTDMT_freeDCtx(ZSTDMT_DCtx * ctx);
#if defined (__cplusplus)
}
#endif
#endif /* ZSTDMT_H */

61
C/zstdmt/zstdmt_common.c Normal file
View File

@@ -0,0 +1,61 @@
/**
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*/
#include "zstd.h"
#include "zstdmt.h"
/* ****************************************
* ZSTDMT Error Management
******************************************/
size_t zstdmt_errcode;
/**
* ZSTDMT_isError() - tells if a return value is an error code
*/
unsigned ZSTDMT_isError(size_t code)
{
return (code > ZSTDMT_ERROR(maxCode));
}
/**
* LZ4MT_getErrorString() - give error code string from function result
*/
const char *ZSTDMT_getErrorString(size_t code)
{
if (ZSTD_isError(zstdmt_errcode))
return ZSTD_getErrorName(zstdmt_errcode);
static const char *notErrorCode = "Unspecified error zstmt code";
switch ((ZSTDMT_ErrorCode) (0 - code)) {
case ZSTDMT_PREFIX(no_error):
return "No error detected";
case ZSTDMT_PREFIX(memory_allocation):
return "Allocation error : not enough memory";
case ZSTDMT_PREFIX(read_fail):
return "Read failure";
case ZSTDMT_PREFIX(write_fail):
return "Write failure";
case ZSTDMT_PREFIX(data_error):
return "Malformed input";
case ZSTDMT_PREFIX(frame_compress):
return "Could not compress frame at once";
case ZSTDMT_PREFIX(frame_decompress):
return "Could not decompress frame at once";
case ZSTDMT_PREFIX(compressionParameter_unsupported):
return "Compression parameter is out of bound";
case ZSTDMT_PREFIX(compression_library):
return "Compression library reports failure";
case ZSTDMT_PREFIX(maxCode):
default:
return notErrorCode;
}
}

427
C/zstdmt/zstdmt_compress.c Normal file
View File

@@ -0,0 +1,427 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdio.h>
#include <stdlib.h>
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "mem.h"
#include "threading.h"
#include "list.h"
#include "zstdmt.h"
/**
* multi threaded zstd compression
*
* - each thread works on his own
* - needs a callback for reading / writing
* - each worker does this:
* 1) get read mutex and read some input
* 2) release read mutex and do compression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
/* worker for compression */
typedef struct {
ZSTDMT_CCtx *ctx;
pthread_t pthread;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
ZSTDMT_Buffer out;
struct list_head node;
};
struct ZSTDMT_CCtx_s {
/* level: 1..ZSTDMT_LEVEL_MAX */
int level;
/* threads: 1..ZSTDMT_THREAD_MAX */
int threads;
/* buffersize for reading input */
int inputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* error handling */
pthread_mutex_t error_mutex;
size_t zstd_errcode;
size_t zstdmt_errcode;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Compression
****************************************/
ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize)
{
ZSTDMT_CCtx *ctx;
int t;
/* allocate ctx */
ctx = (ZSTDMT_CCtx *) malloc(sizeof(ZSTDMT_CCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > ZSTDMT_THREAD_MAX)
goto err_ctx;
/* check level */
if (level < 1 || level > ZSTDMT_LEVEL_MAX)
goto err_ctx;
/* calculate chunksize for one thread */
if (inputsize)
ctx->inputsize = inputsize;
else {
/* XXX - windowlog */ const int mb[] = {
2, 2, 4, 4, 6, 6, 6, /* 1 - 7 */
8, 8, 8, 8, 8, 8, 8, /* 8 - 14 */
16, 16, 16, 16, 16, 16, 16, 16 /* 15 - 22 */
};
ctx->inputsize = 1024 * 1024 * mb[level - 1];
}
/* setup ctx */
ctx->level = level;
ctx->threads = threads;
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
pthread_mutex_init(&ctx->error_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
if (!ctx->cwork)
goto err_ctx;
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
w->ctx = ctx;
}
return ctx;
err_ctx:
free(ctx);
return 0;
}
/**
* pt_write - queue for compressed output
*/
static size_t pt_write(ZSTDMT_CCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
int rv;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
/* write zero byte frame (9 bytes) for type identification */
if (unlikely(wl->frame == 0)) {
unsigned char frame0[] =
{ 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x48, 0x01, 0x00, 0x00 };
ZSTDMT_Buffer b;
b.buf = frame0;
b.size = 9;
rv = ctx->fn_write(ctx->arg_write, &b);
if (rv == -1)
return ZSTDMT_ERROR(write_fail);
if (b.size != 9)
return ZSTDMT_ERROR(write_fail);
ctx->outsize += 9;
}
/* the entry isn't the currently needed, return... */
if (wl->frame != ctx->curframe)
return 0;
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv == -1)
return ZSTDMT_ERROR(write_fail);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
/* parallel compression worker */
static void *pt_compress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
ZSTDMT_CCtx *ctx = w->ctx;
struct writelist *wl;
size_t result;
ZSTDMT_Buffer in;
/* inbuf is constant */
in.size = ctx->inputsize;
in.buf = malloc(in.size);
if (!in.buf)
return (void *)ZSTDMT_ERROR(memory_allocation);
for (;;) {
struct list_head *entry;
ZSTDMT_Buffer *out;
int rv;
/* allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
pthread_mutex_unlock(&ctx->write_mutex);
free(in.buf);
return (void *)ZSTDMT_ERROR(memory_allocation);
}
wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;;
wl->out.buf = malloc(wl->out.size);
if (!wl->out.buf) {
pthread_mutex_unlock(&ctx->write_mutex);
free(in.buf);
return (void *)ZSTDMT_ERROR(memory_allocation);
}
list_add(&wl->node, &ctx->writelist_busy);
}
pthread_mutex_unlock(&ctx->write_mutex);
out = &wl->out;
/* read new input */
pthread_mutex_lock(&ctx->read_mutex);
in.size = ctx->inputsize;
rv = ctx->fn_read(ctx->arg_read, &in);
if (rv == -1) {
pthread_mutex_unlock(&ctx->read_mutex);
result = ZSTDMT_ERROR(read_fail);
goto error;
}
/* eof */
if (in.size == 0) {
free(in.buf);
pthread_mutex_unlock(&ctx->read_mutex);
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
goto okay;
}
ctx->insize += in.size;
wl->frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* compress whole frame */
{
unsigned char *outbuf = out->buf;
result = ZSTD_compress(outbuf + 12, out->size - 12, in.buf, in.size, ctx->level);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
result = ZSTDMT_ERROR(compression_library);
goto error;
}
}
/* write skippable frame */
{
unsigned char *outbuf = out->buf;
MEM_writeLE32(outbuf + 0, ZSTDMT_MAGIC_SKIPPABLE);
MEM_writeLE32(outbuf + 4, 4);
MEM_writeLE32(outbuf + 8, (U32) result);
out->size = result + 12;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
pthread_mutex_unlock(&ctx->write_mutex);
if (ZSTDMT_isError(result))
goto error;
}
okay:
return 0;
error:
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
return (void *)result;
}
/* compress data, until input ends */
size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx * ctx, ZSTDMT_RdWr_t * rdwr)
{
int t;
void *rv = 0;
if (!ctx)
return ZSTDMT_ERROR(init_missing);
/* setup reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/* init counter and error codes */
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
ctx->zstdmt_errcode = 0;
/* start all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
pthread_create(&w->pthread, NULL, pt_compress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
if (p)
rv = p;
}
/* clean up the free list */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
/* on error, these two lists may have some entries */
if (rv) {
struct writelist *wl;
struct list_head *entry;
while (!list_empty(&ctx->writelist_busy)) {
entry = list_first(&ctx->writelist_busy);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
while (!list_empty(&ctx->writelist_done)) {
entry = list_first(&ctx->writelist_done);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
}
return (size_t) rv;
}
/* returns current uncompressed data size */
size_t ZSTDMT_GetInsizeCCtx(ZSTDMT_CCtx * ctx)
{
if (!ctx)
return ZSTDMT_ERROR(init_missing);
/* no mutex needed here */
return ctx->insize;
}
/* returns the current compressed data size */
size_t ZSTDMT_GetOutsizeCCtx(ZSTDMT_CCtx * ctx)
{
if (!ctx)
return ZSTDMT_ERROR(init_missing);
/* no mutex needed here */
return ctx->outsize;
}
/* returns the current compressed data frame count */
size_t ZSTDMT_GetFramesCCtx(ZSTDMT_CCtx * ctx)
{
if (!ctx)
return ZSTDMT_ERROR(init_missing);
/* no mutex needed here */
return ctx->curframe;
}
/* free all allocated buffers and structures */
void ZSTDMT_freeCCtx(ZSTDMT_CCtx * ctx)
{
if (!ctx)
return;
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

857
C/zstdmt/zstdmt_decompress.c Normal file
View File

@@ -0,0 +1,857 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#include <stdlib.h>
#include <string.h>
#define ZSTD_STATIC_LINKING_ONLY
#include "zstd.h"
#include "mem.h"
#include "threading.h"
#include "list.h"
#include "zstdmt.h"
/**
* multi threaded zstd decompression
*
* - each thread works on his own
* - needs a callback for reading / writing
* - each worker does this:
* 1) get read mutex and read some input
* 2) release read mutex and do decompression
* 3) get write mutex and write result
* 4) begin with step 1 again, until no input
*/
#if 0
#include <stdio.h>
#define dprintf(fmt, arg...) do { printf(fmt, ## arg); } while (0)
#else
#define dprintf(fmt, ...)
#endif /* DEBUG */
extern size_t zstdmt_errcode;
/* worker for compression */
typedef struct {
ZSTDMT_DCtx *ctx;
pthread_t pthread;
ZSTDMT_Buffer in;
ZSTD_DStream *dctx;
} cwork_t;
struct writelist;
struct writelist {
size_t frame;
ZSTDMT_Buffer out;
struct list_head node;
};
struct ZSTDMT_DCtx_s {
/* threads: 1..ZSTDMT_THREAD_MAX */
int threads;
int threadswanted;
/* input buffer, used at single threading */
size_t inputsize;
/* buffersize used for output */
size_t outputsize;
/* statistic */
size_t insize;
size_t outsize;
size_t curframe;
size_t frames;
/* threading */
cwork_t *cwork;
/* reading input */
pthread_mutex_t read_mutex;
fn_read *fn_read;
void *arg_read;
/* writing output */
pthread_mutex_t write_mutex;
fn_write *fn_write;
void *arg_write;
/* error handling */
pthread_mutex_t error_mutex;
/* lists for writing queue */
struct list_head writelist_free;
struct list_head writelist_busy;
struct list_head writelist_done;
};
/* **************************************
* Decompression
****************************************/
ZSTDMT_DCtx *ZSTDMT_createDCtx(int threads, int inputsize)
{
ZSTDMT_DCtx *ctx;
/* allocate ctx */
ctx = (ZSTDMT_DCtx *) malloc(sizeof(ZSTDMT_DCtx));
if (!ctx)
return 0;
/* check threads value */
if (threads < 1 || threads > ZSTDMT_THREAD_MAX)
return 0;
/* setup ctx */
ctx->threadswanted = threads;
ctx->threads = 0;
ctx->insize = 0;
ctx->outsize = 0;
ctx->frames = 0;
ctx->curframe = 0;
/* will be used for single stream only */
if (inputsize)
ctx->inputsize = inputsize;
else
ctx->inputsize = 1024 * 512;
/* frame size (will get higher, when needed) */
ctx->outputsize = 1024 * 512;
/* later */
ctx->cwork = 0;
return ctx;
}
/**
* IsZstd_Magic - check, if 4 bytes are valid ZSTD MAGIC
*/
static int IsZstd_Magic(unsigned char *buf)
{
U32 magic = MEM_readLE32(buf);
if (magic == ZSTDMT_MAGICNUMBER_V01)
return 1;
return (magic >= ZSTDMT_MAGICNUMBER_MIN
&& magic <= ZSTDMT_MAGICNUMBER_MAX);
}
/**
* IsZstd_Skippable - check, if 4 bytes are MAGIC_SKIPPABLE
*/
static int IsZstd_Skippable(unsigned char *buf)
{
return (MEM_readLE32(buf) == ZSTDMT_MAGIC_SKIPPABLE);
}
/**
* pt_write - queue for decompressed output
*/
static size_t pt_write(ZSTDMT_DCtx * ctx, struct writelist *wl)
{
struct list_head *entry;
/* move the entry to the done list */
list_move(&wl->node, &ctx->writelist_done);
again:
/* check, what can be written ... */
list_for_each(entry, &ctx->writelist_done) {
wl = list_entry(entry, struct writelist, node);
if (wl->frame == ctx->curframe) {
int rv = ctx->fn_write(ctx->arg_write, &wl->out);
if (rv == -1)
return ZSTDMT_ERROR(write_fail);
ctx->outsize += wl->out.size;
ctx->curframe++;
list_move(entry, &ctx->writelist_free);
goto again;
}
}
return 0;
}
/**
* pt_read - read compressed input
*/
static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame)
{
unsigned char hdrbuf[12];
ZSTDMT_Buffer hdr;
size_t toRead;
int rv;
pthread_mutex_lock(&ctx->read_mutex);
/* special case, some bytes were read by magic check */
if (unlikely(ctx->frames == 0)) {
/* the magic check reads exactly 16 bytes! */
if (unlikely(in->size != 16))
goto error_data;
ctx->insize += 16;
/**
* zstdmt mode, with zstd magic prefix
* 9 bytes zero byte frame + 12 byte skippable
* - 21 bytes to read, 16 bytes done
* - read 5 bytes, put them together (12 byte hdr)
*/
if (!IsZstd_Skippable(in->buf)) {
memcpy(hdrbuf, in->buf, 7);
hdr.buf = hdrbuf + 7;
hdr.size = 5;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv == -1)
goto error_read;
if (hdr.size != 5)
goto error_data;
hdr.buf = hdrbuf;
ctx->insize += 16 + 5;
/* read data */
toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
in->size = toRead;
in->buf = malloc(in->size);
if (!in->buf)
goto error_nomem;
in->allocated = in->size;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1)
goto error_read;
if (in->size != toRead)
goto error_data;
ctx->insize += in->size;
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
return 0; /* done! */
}
/**
* pzstd mode, no prefix
* - start directly with 12 byte skippable frame
*/
if (IsZstd_Skippable(in->buf)) {
unsigned char *start = in->buf; /* 16 bytes data */
toRead = MEM_readLE32((unsigned char *)start + 8);
in->size = toRead;
in->buf = malloc(in->size);
if (!in->buf)
goto error_nomem;
in->allocated = in->size;
/* copy 4 bytes user data to new buf */
memcpy(in->buf, start + 12, 4);
start = in->buf; /* point to in->buf now */
/* 12 byte skippable, so 4 bytes data done */
in->buf = start + 4;
in->size = toRead - 4;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1)
goto error_read;
if (in->size != toRead - 4)
goto error_data;
ctx->insize += in->size;
in->buf = start; /* restore inbuf */
in->size += 4;
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
return 0; /* done! */
}
}
/**
* read next skippable frame (12 bytes)
* 4 bytes skippable magic
* 4 bytes little endian, must be: 4 (user data size)
* 4 bytes little endian, size to read (user data)
*/
hdr.buf = hdrbuf;
hdr.size = 12;
rv = ctx->fn_read(ctx->arg_read, &hdr);
if (rv == -1)
goto error_read;
/* eof reached ? */
if (unlikely(hdr.size == 0)) {
pthread_mutex_unlock(&ctx->read_mutex);
in->size = 0;
return 0;
}
/* check header data */
if (unlikely(hdr.size != 12))
goto error_read;
if (unlikely(!IsZstd_Skippable(hdr.buf)))
goto error_data;
ctx->insize += 12;
/* read new input (size should be _toRead_ bytes */
toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
{
if (in->allocated < toRead) {
/* need bigger input buffer */
if (in->allocated)
in->buf = realloc(in->buf, toRead);
else
in->buf = malloc(toRead);
if (!in->buf)
goto error_nomem;
in->allocated = toRead;
}
in->size = toRead;
rv = ctx->fn_read(ctx->arg_read, in);
/* generic read failure! */
if (rv == -1)
goto error_read;
/* needed more bytes! */
if (in->size != toRead)
goto error_data;
ctx->insize += in->size;
}
*frame = ctx->frames++;
pthread_mutex_unlock(&ctx->read_mutex);
/* done, no error */
return 0;
error_data:
pthread_mutex_unlock(&ctx->read_mutex);
return ZSTDMT_ERROR(data_error);
error_read:
pthread_mutex_unlock(&ctx->read_mutex);
return ZSTDMT_ERROR(read_fail);
error_nomem:
pthread_mutex_unlock(&ctx->read_mutex);
return ZSTDMT_ERROR(memory_allocation);
}
static void *pt_decompress(void *arg)
{
cwork_t *w = (cwork_t *) arg;
ZSTDMT_Buffer *in = &w->in;
ZSTDMT_DCtx *ctx = w->ctx;
struct writelist *wl;
size_t result = 0;
ZSTDMT_Buffer collect;
/* init dstream stream */
result = ZSTD_initDStream(w->dctx);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
return (void *)ZSTDMT_ERROR(compression_library);
}
collect.buf = 0;
collect.size = 0;
collect.allocated = 0;
for (;;) {
ZSTDMT_Buffer *out;
ZSTD_inBuffer zIn;
ZSTD_outBuffer zOut;
/* select or allocate space for new output */
pthread_mutex_lock(&ctx->write_mutex);
if (!list_empty(&ctx->writelist_free)) {
/* take unused entry */
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
list_move(entry, &ctx->writelist_busy);
} else {
/* allocate new one */
wl = (struct writelist *)
malloc(sizeof(struct writelist));
if (!wl) {
result = ZSTDMT_ERROR(memory_allocation);
goto error_unlock;
}
out = &wl->out;
out->size = ctx->outputsize;
out->buf = malloc(out->size);
if (!out->buf) {
result = ZSTDMT_ERROR(memory_allocation);
goto error_unlock;
}
out->allocated = out->size;
list_add(&wl->node, &ctx->writelist_busy);
}
/* start with 512KB */
/* XXX, add framesize detection... */
out = &wl->out;
pthread_mutex_unlock(&ctx->write_mutex);
/* init dstream stream */
result = ZSTD_resetDStream(w->dctx);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
return (void *)ZSTDMT_ERROR(compression_library);
}
/* zero should not happen here! */
result = pt_read(ctx, in, &wl->frame);
if (in->size == 0)
break;
if (ZSTDMT_isError(result)) {
goto error_lock;
}
zIn.size = in->allocated;
zIn.src = in->buf;
zIn.pos = 0;
for (;;) {
again:
/* decompress loop */
zOut.size = out->allocated;
zOut.dst = out->buf;
zOut.pos = 0;
dprintf
("ZSTD_decompressStream() zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n",
zIn.size, zIn.pos, zOut.size, zOut.pos);
result = ZSTD_decompressStream(w->dctx, &zOut, &zIn);
dprintf
("ZSTD_decompressStream(), ret=%zu zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n",
result, zIn.size, zIn.pos, zOut.size, zOut.pos);
if (ZSTD_isError(result))
goto error_clib;
/* end of frame */
if (result == 0) {
/* put collected stuff together */
if (collect.size) {
void *bnew;
bnew = malloc(collect.size + zOut.pos);
if (!bnew) {
result =
ZSTDMT_ERROR(memory_allocation);
goto error_lock;
}
memcpy((char *)bnew, collect.buf,
collect.size);
memcpy((char *)bnew + collect.size,
out->buf, zOut.pos);
free(collect.buf);
free(out->buf);
out->buf = bnew;
out->size = collect.size + zOut.pos;
out->allocated = out->size;
collect.buf = 0;
collect.size = 0;
} else {
out->size = zOut.pos;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);
result = pt_write(ctx, wl);
if (ZSTDMT_isError(result))
goto error_unlock;
pthread_mutex_unlock(&ctx->write_mutex);
/* will read next input */
break;
}
/* out buffer to small for full frame */
if (result != 0) {
/* collect old content from out */
collect.buf =
realloc(collect.buf,
collect.size + out->size);
memcpy((char *)collect.buf + collect.size,
out->buf, out->size);
collect.size = collect.size + out->size;
/* double the buffer, until it fits */
pthread_mutex_lock(&ctx->write_mutex);
out->size *= 2;
ctx->outputsize = out->size;
pthread_mutex_unlock(&ctx->write_mutex);
out->buf = realloc(out->buf, out->size);
if (!out->buf) {
result = ZSTDMT_ERROR(memory_allocation);
goto error_lock;
}
out->allocated = out->size;
goto again;
}
if (zIn.pos == zIn.size)
break; /* should fail... */
} /* decompress loop */
} /* read input loop */
/* everything is okay */
pthread_mutex_lock(&ctx->write_mutex);
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return 0;
error_clib:
zstdmt_errcode = result;
result = ZSTDMT_ERROR(compression_library);
/* fall through */
error_lock:
pthread_mutex_lock(&ctx->write_mutex);
error_unlock:
list_move(&wl->node, &ctx->writelist_free);
pthread_mutex_unlock(&ctx->write_mutex);
if (in->allocated)
free(in->buf);
return (void *)result;
}
/* single threaded */
static size_t st_decompress(void *arg)
{
ZSTDMT_DCtx *ctx = (ZSTDMT_DCtx *) arg;
cwork_t *w = &ctx->cwork[0];
ZSTDMT_Buffer In, Out;
ZSTDMT_Buffer *in = &In;
ZSTDMT_Buffer *out = &Out;
ZSTDMT_Buffer *magic = &w->in;
size_t result;
int rv;
ZSTD_inBuffer zIn;
ZSTD_outBuffer zOut;
/* init dstream stream */
result = ZSTD_initDStream(w->dctx);
if (ZSTD_isError(result)) {
zstdmt_errcode = result;
return ZSTDMT_ERROR(compression_library);
}
/* allocate space for input buffer */
in->size = ZSTD_DStreamInSize();
in->buf = malloc(in->size);
if (!in->buf)
return ZSTDMT_ERROR(memory_allocation);
in->allocated = in->size;
/* allocate space for output buffer */
out->size = ZSTD_DStreamOutSize();
out->buf = malloc(out->size);
if (!out->buf) {
free(in->buf);
return ZSTDMT_ERROR(memory_allocation);
}
out->allocated = out->size;
/* we read already some bytes, handle that: */
{
/* remember in->buf */
unsigned char *buf = in->buf;
/* fill first read bytes to buffer... */
memcpy(in->buf, magic->buf, magic->size);
magic->buf = in->buf;
in->buf = buf + magic->size;
in->size = in->allocated - magic->size;
/* read more bytes, to fill buffer */
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1) {
result = ZSTDMT_ERROR(read_fail);
goto error;
}
/* ready, first buffer complete */
in->buf = buf;
in->size += magic->size;
ctx->insize += in->size;
}
zIn.src = in->buf;
zIn.size = in->size;
zIn.pos = 0;
zOut.dst = out->buf;
for (;;) {
for (;;) {
/* decompress loop */
zOut.size = out->allocated;
zOut.pos = 0;
result = ZSTD_decompressStream(w->dctx, &zOut, &zIn);
if (ZSTD_isError(result))
goto error_clib;
if (zOut.pos) {
ZSTDMT_Buffer w;
w.size = zOut.pos;
w.buf = zOut.dst;
rv = ctx->fn_write(ctx->arg_write, &w);
ctx->outsize += zOut.pos;
}
/* one more round */
if ((zIn.pos == zIn.size) && (result == 1) && zOut.pos)
continue;
/* finished */
if (zIn.pos == zIn.size)
break;
/* end of frame */
if (result == 0) {
result = ZSTD_resetDStream(w->dctx);
if (ZSTD_isError(result))
goto error_clib;
}
} /* decompress */
/* read next input */
in->size = in->allocated;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1) {
result = ZSTDMT_ERROR(read_fail);
goto error;
}
if (in->size == 0)
goto okay;
ctx->insize += in->size;
zIn.size = in->size;
zIn.pos = 0;
} /* read */
error_clib:
zstdmt_errcode = result;
result = ZSTDMT_ERROR(compression_library);
/* fall through */
error:
/* return with error */
free(out->buf);
free(in->buf);
return result;
okay:
/* no error */
free(out->buf);
free(in->buf);
return 0;
}
#define TYPE_UNKNOWN 0
#define TYPE_SINGLE_THREAD 1
#define TYPE_MULTI_THREAD 2
size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr)
{
unsigned char buf[16];
ZSTDMT_Buffer In;
ZSTDMT_Buffer *in = &In;
cwork_t *w;
int t, rv, type = TYPE_UNKNOWN;
if (!ctx)
return ZSTDMT_ERROR(compressionParameter_unsupported);
/* init reading and writing functions */
ctx->fn_read = rdwr->fn_read;
ctx->fn_write = rdwr->fn_write;
ctx->arg_read = rdwr->arg_read;
ctx->arg_write = rdwr->arg_write;
/**
* possible valid magic's for us, we need 16 bytes, for checking
*
* 1) ZSTDMT_MAGIC @0 -> ST Stream
* 2) ZSTDMT_MAGIC @0 + MAGIC_SKIPPABLE @9 -> MT Stream else ST
* 3) MAGIC_SKIPPABLE @0 + ZSTDMT_MAGIC @12 -> MT Stream
* 4) all other: not valid!
*/
/* check for ZSTDMT_MAGIC_SKIPPABLE */
in->buf = buf;
in->size = 16;
rv = ctx->fn_read(ctx->arg_read, in);
if (rv == -1)
return ZSTDMT_ERROR(read_fail);
/* must be single threaded standard zstd, when smaller 16 bytes */
if (in->size < 16) {
if (!IsZstd_Magic(buf))
return ZSTDMT_ERROR(data_error);
dprintf("single thread style, current pos=%zu\n", in->size);
type = TYPE_SINGLE_THREAD;
if (in->size == 9) {
/* create empty file */
ctx->threads = 0;
ctx->cwork = 0;
return 0;
}
} else {
if (IsZstd_Skippable(buf) && IsZstd_Magic(buf + 12)) {
/* pzstd */
dprintf("pzstd style\n");
type = TYPE_MULTI_THREAD;
} else if (IsZstd_Magic(buf) && IsZstd_Skippable(buf + 9)) {
/* zstdmt */
dprintf("zstdmt style\n");
type = TYPE_MULTI_THREAD;
/* set buffer to the */
} else if (IsZstd_Magic(buf)) {
/* some std zstd stream */
dprintf("single thread style, current pos=%zu\n",
in->size);
type = TYPE_SINGLE_THREAD;
} else {
/* invalid */
dprintf("not valid\n");
return ZSTDMT_ERROR(data_error);
}
}
/* use single thread extraction, when only one thread is there */
if (ctx->threadswanted == 1)
type = TYPE_SINGLE_THREAD;
/* single threaded, but with known sizes */
if (type == TYPE_SINGLE_THREAD) {
ctx->threads = 1;
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t));
if (!ctx->cwork)
return ZSTDMT_ERROR(memory_allocation);
w = &ctx->cwork[0];
w->in.buf = in->buf;
w->in.size = in->size;
w->in.allocated = 0;
w->ctx = ctx;
w->dctx = ZSTD_createDStream();
if (!w->dctx)
return ZSTDMT_ERROR(memory_allocation);
/* test, if pt_decompress is better... */
return st_decompress(ctx);
}
/* setup thread work */
ctx->threads = ctx->threadswanted;
ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * ctx->threads);
if (!ctx->cwork)
return ZSTDMT_ERROR(memory_allocation);
for (t = 0; t < ctx->threads; t++) {
w = &ctx->cwork[t];
/* one of the threads must reuse the first bytes */
w->in.buf = in->buf;
w->in.size = in->size;
w->in.allocated = 0;
w->ctx = ctx;
w->dctx = ZSTD_createDStream();
if (!w->dctx)
return ZSTDMT_ERROR(memory_allocation);
}
/* real multi threaded, init pthread's */
pthread_mutex_init(&ctx->read_mutex, NULL);
pthread_mutex_init(&ctx->write_mutex, NULL);
pthread_mutex_init(&ctx->error_mutex, NULL);
INIT_LIST_HEAD(&ctx->writelist_free);
INIT_LIST_HEAD(&ctx->writelist_busy);
INIT_LIST_HEAD(&ctx->writelist_done);
/* multi threaded */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
pthread_create(&w->pthread, NULL, pt_decompress, w);
}
/* wait for all workers */
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
void *p;
pthread_join(w->pthread, &p);
if (p)
return (size_t) p;
}
/* clean up pthread stuff */
pthread_mutex_destroy(&ctx->read_mutex);
pthread_mutex_destroy(&ctx->write_mutex);
pthread_mutex_destroy(&ctx->error_mutex);
/* clean up the buffers */
while (!list_empty(&ctx->writelist_free)) {
struct writelist *wl;
struct list_head *entry;
entry = list_first(&ctx->writelist_free);
wl = list_entry(entry, struct writelist, node);
free(wl->out.buf);
list_del(&wl->node);
free(wl);
}
return 0;
}
/* returns current uncompressed data size */
size_t ZSTDMT_GetInsizeDCtx(ZSTDMT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->insize;
}
/* returns the current compressed data size */
size_t ZSTDMT_GetOutsizeDCtx(ZSTDMT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->outsize;
}
/* returns the current compressed frames */
size_t ZSTDMT_GetFramesDCtx(ZSTDMT_DCtx * ctx)
{
if (!ctx)
return 0;
return ctx->curframe;
}
void ZSTDMT_freeDCtx(ZSTDMT_DCtx * ctx)
{
int t;
if (!ctx)
return;
for (t = 0; t < ctx->threads; t++) {
cwork_t *w = &ctx->cwork[t];
ZSTD_freeDStream(w->dctx);
}
if (ctx->cwork)
free(ctx->cwork);
free(ctx);
ctx = 0;
return;
}

27
CPP/7zip/7zip.mak
View File

@@ -27,7 +27,10 @@ OBJS = \
$(COMPRESS_OBJS) \
$(CRYPTO_OBJS) \
$(C_OBJS) \
$(LZ4_OBJS) \
$(LZ5_OBJS) \
$(ZSTD_OBJS) \
$(ZSTDMT_OBJS) \
$(ASM_OBJS) \
$O\resource.res \
@@ -173,11 +176,26 @@ $(C_OBJS): ../../../../C/$(*B).c
$(COMPL_O2)
!ENDIF
!IFDEF LZ4_OBJS
$(LZ4_OBJS): ../../../../C/lz4/$(*B).c
$(COMPL_O2)
!ENDIF
!IFDEF LZ5_OBJS
$(LZ5_OBJS): ../../../../C/lz5/$(*B).c
$(COMPL_O2)
!ENDIF
!IFDEF ZSTD_OBJS
$(ZSTD_OBJS): ../../../../C/zstd/$(*B).c
$(COMPL_O2)
!ENDIF
!IFDEF ZSTDMT_OBJS
$(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c
$(COMPL_O2)
!ENDIF
!ELSE
@@ -240,8 +258,17 @@ $(ZSTD_OBJS): ../../../../C/zstd/$(*B).c
$(COMPLB_O2)
{../../../../C}.c{$O}.obj::
$(CCOMPLB)
{../../../../C/lz4}.c{$O}.obj::
$(CCOMPLB)
{../../../../C/lz5}.c{$O}.obj::
$(CCOMPLB)
{../../../../C/zstd}.c{$O}.obj::
$(CCOMPLB)
{../../../../C/zstdmt}.c{$O}.obj::
$(CCOMPLB) \
-I ../../../../C/lz4 \
-I ../../../../C/lz5 \
-I ../../../../C/zstd
!ENDIF

2
CPP/7zip/Archive/7z/7zUpdate.cpp
View File

@@ -558,7 +558,7 @@ static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param
}
static const char *g_Exts =
" 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
" 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lz tlz lz4 tlz4 lz5 tlz5 lzh lzo lzx pak rar rpm sit zoo zst"
" zip jar ear war msi"
" 3gp avi mov mpeg mpg mpe wmv"
" aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"

2
CPP/7zip/Archive/IArchive.h
View File

@@ -155,7 +155,7 @@ IArchiveExtractCallback::GetStream()
if (IProgress::SetTotal() was called)
{
IProgress::SetCompleted(completeValue) uses
packSize - for some stream formats (xz, gz, bz2, lzma, z, ppmd).
packSize - for some stream formats (xz, gz, bz2, lz, lzma, z, ppmd).
unpackSize - for another formats.
}
else

BIN
CPP/7zip/Archive/Icons/lz4.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.6 KiB

BIN
CPP/7zip/Archive/Icons/lz5.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.6 KiB

366
CPP/7zip/Archive/Lz4Handler.cpp Normal file
View File

@@ -0,0 +1,366 @@
// ZstdHandler.cpp
#include "StdAfx.h"
#include "../../../C/CpuArch.h"
#include "../../Common/ComTry.h"
#include "../../Common/Defs.h"
#include "../Common/ProgressUtils.h"
#include "../Common/RegisterArc.h"
#include "../Common/StreamUtils.h"
#include "../Compress/Lz4Decoder.h"
#include "../Compress/Lz4Encoder.h"
#include "../Compress/CopyCoder.h"
#include "Common/DummyOutStream.h"
#include "Common/HandlerOut.h"
using namespace NWindows;
namespace NArchive {
namespace NLZ4 {
class CHandler:
public IInArchive,
public IArchiveOpenSeq,
public IOutArchive,
public ISetProperties,
public CMyUnknownImp
{
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
bool _isArc;
bool _dataAfterEnd;
bool _needMoreInput;
bool _packSize_Defined;
bool _unpackSize_Defined;
UInt64 _packSize;
UInt64 _unpackSize;
UInt64 _numStreams;
UInt64 _numBlocks;
CSingleMethodProps _props;
public:
MY_UNKNOWN_IMP4(
IInArchive,
IArchiveOpenSeq,
IOutArchive,
ISetProperties)
INTERFACE_IInArchive(;)
INTERFACE_IOutArchive(;)
STDMETHOD(OpenSeq)(ISequentialInStream *stream);
STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
CHandler() { }
};
static const Byte kProps[] =
{
kpidSize,
kpidPackSize
};
static const Byte kArcProps[] =
{
kpidNumStreams,
kpidNumBlocks
};
IMP_IInArchive_Props
IMP_IInArchive_ArcProps
STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/)
{
return S_OK;
}
STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
{
*numItems = 1;
return S_OK;
}
STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
{
NCOM::CPropVariant prop;
switch (propID)
{
case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
}
prop.Detach(value);
return S_OK;
}
static const unsigned kSignatureCheckSize = 4;
API_FUNC_static_IsArc IsArc_lz4(const Byte *p, size_t size)
{
if (size < 4)
return k_IsArc_Res_NEED_MORE;
UInt32 magic = GetUi32(p);
// skippable frames
if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) {
if (size < 16)
return k_IsArc_Res_NEED_MORE;
magic = GetUi32(p+12);
}
// lz4 magic
if (magic == 0x184D2204)
return k_IsArc_Res_YES;
return k_IsArc_Res_NO;
}
}
STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)
{
COM_TRY_BEGIN
Close();
{
Byte buf[kSignatureCheckSize];
RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize));
if (IsArc_lz4(buf, kSignatureCheckSize) == k_IsArc_Res_NO)
return S_FALSE;
_isArc = true;
_stream = stream;
_seqStream = stream;
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
}
return S_OK;
COM_TRY_END
}
STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
{
Close();
_isArc = true;
_seqStream = stream;
return S_OK;
}
STDMETHODIMP CHandler::Close()
{
_isArc = false;
_dataAfterEnd = false;
_needMoreInput = false;
_packSize_Defined = false;
_unpackSize_Defined = false;
_packSize = 0;
_seqStream.Release();
_stream.Release();
return S_OK;
}
STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
Int32 testMode, IArchiveExtractCallback *extractCallback)
{
COM_TRY_BEGIN
if (numItems == 0)
return S_OK;
if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
return E_INVALIDARG;
if (_packSize_Defined)
extractCallback->SetTotal(_packSize);
CMyComPtr<ISequentialOutStream> realOutStream;
Int32 askMode = testMode ?
NExtract::NAskMode::kTest :
NExtract::NAskMode::kExtract;
RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
if (!testMode && !realOutStream)
return S_OK;
extractCallback->PrepareOperation(askMode);
Int32 opRes;
{
NCompress::NLZ4::CDecoder *decoderSpec = new NCompress::NLZ4::CDecoder;
CMyComPtr<ICompressCoder> decoder = decoderSpec;
decoderSpec->SetInStream(_seqStream);
CDummyOutStream *outStreamSpec = new CDummyOutStream;
CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
outStreamSpec->SetStream(realOutStream);
outStreamSpec->Init();
realOutStream.Release();
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(extractCallback, true);
UInt64 packSize = 0;
UInt64 unpackedSize = 0;
HRESULT result = S_OK;
for (;;)
{
lps->InSize = packSize;
lps->OutSize = unpackedSize;
RINOK(lps->SetCur());
result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
UInt64 streamSize = decoderSpec->GetInputProcessedSize();
if (result != S_FALSE && result != S_OK)
return result;
if (unpackedSize == 0)
break;
if (streamSize == packSize)
{
// no new bytes in input stream, So it's good end of archive.
result = S_OK;
break;
}
if (packSize > streamSize)
return E_FAIL;
if (result != S_OK)
break;
}
decoderSpec->ReleaseInStream();
outStream.Release();
if (!_isArc)
opRes = NExtract::NOperationResult::kIsNotArc;
else if (_needMoreInput)
opRes = NExtract::NOperationResult::kUnexpectedEnd;
else if (_dataAfterEnd)
opRes = NExtract::NOperationResult::kDataAfterEnd;
else if (result == S_FALSE)
opRes = NExtract::NOperationResult::kDataError;
else if (result == S_OK)
opRes = NExtract::NOperationResult::kOK;
else
return result;
}
return extractCallback->SetOperationResult(opRes);
COM_TRY_END
}
static HRESULT UpdateArchive(
UInt64 unpackSize,
ISequentialOutStream *outStream,
const CProps &props,
IArchiveUpdateCallback *updateCallback)
{
RINOK(updateCallback->SetTotal(unpackSize));
CMyComPtr<ISequentialInStream> fileInStream;
RINOK(updateCallback->GetStream(0, &fileInStream));
CLocalProgress *localProgressSpec = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
localProgressSpec->Init(updateCallback, true);
NCompress::NLZ4::CEncoder *encoderSpec = new NCompress::NLZ4::CEncoder;
CMyComPtr<ICompressCoder> encoder = encoderSpec;
RINOK(props.SetCoderProps(encoderSpec, NULL));
RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));
return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
}
STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
{
*type = NFileTimeType::kUnix;
return S_OK;
}
STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
IArchiveUpdateCallback *updateCallback)
{
COM_TRY_BEGIN
if (numItems != 1)
return E_INVALIDARG;
Int32 newData, newProps;
UInt32 indexInArchive;
if (!updateCallback)
return E_FAIL;
RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive));
if ((newProps))
{
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop));
if (prop.vt != VT_EMPTY)
if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
return E_INVALIDARG;
}
}
if ((newData))
{
UInt64 size;
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
if (prop.vt != VT_UI8)
return E_INVALIDARG;
size = prop.uhVal.QuadPart;
}
return UpdateArchive(size, outStream, _props, updateCallback);
}
if (indexInArchive != 0)
return E_INVALIDARG;
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(updateCallback, true);
CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
if (opCallback)
{
RINOK(opCallback->ReportOperation(
NEventIndexType::kInArcIndex, 0,
NUpdateNotifyOp::kReplicate))
}
if (_stream)
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
return NCompress::CopyStream(_stream, outStream, progress);
COM_TRY_END
}
STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
{
return _props.SetProperties(names, values, numProps);
}
static const Byte k_Signature[] = "0x184D2204";
REGISTER_ARC_IO(
"lz4", "lz4 tlz4", "* .tar", 0x0f,
k_Signature,
0,
NArcInfoFlags::kKeepName,
IsArc_lz4)
}}

366
CPP/7zip/Archive/Lz5Handler.cpp Normal file
View File

@@ -0,0 +1,366 @@
// ZstdHandler.cpp
#include "StdAfx.h"
#include "../../../C/CpuArch.h"
#include "../../Common/ComTry.h"
#include "../../Common/Defs.h"
#include "../Common/ProgressUtils.h"
#include "../Common/RegisterArc.h"
#include "../Common/StreamUtils.h"
#include "../Compress/Lz5Decoder.h"
#include "../Compress/Lz5Encoder.h"
#include "../Compress/CopyCoder.h"
#include "Common/DummyOutStream.h"
#include "Common/HandlerOut.h"
using namespace NWindows;
namespace NArchive {
namespace NLZ5 {
class CHandler:
public IInArchive,
public IArchiveOpenSeq,
public IOutArchive,
public ISetProperties,
public CMyUnknownImp
{
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
bool _isArc;
bool _dataAfterEnd;
bool _needMoreInput;
bool _packSize_Defined;
bool _unpackSize_Defined;
UInt64 _packSize;
UInt64 _unpackSize;
UInt64 _numStreams;
UInt64 _numBlocks;
CSingleMethodProps _props;
public:
MY_UNKNOWN_IMP4(
IInArchive,
IArchiveOpenSeq,
IOutArchive,
ISetProperties)
INTERFACE_IInArchive(;)
INTERFACE_IOutArchive(;)
STDMETHOD(OpenSeq)(ISequentialInStream *stream);
STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
CHandler() { }
};
static const Byte kProps[] =
{
kpidSize,
kpidPackSize
};
static const Byte kArcProps[] =
{
kpidNumStreams,
kpidNumBlocks
};
IMP_IInArchive_Props
IMP_IInArchive_ArcProps
STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/)
{
return S_OK;
}
STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
{
*numItems = 1;
return S_OK;
}
STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
{
NCOM::CPropVariant prop;
switch (propID)
{
case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
}
prop.Detach(value);
return S_OK;
}
static const unsigned kSignatureCheckSize = 4;
API_FUNC_static_IsArc IsArc_lz5(const Byte *p, size_t size)
{
if (size < 4)
return k_IsArc_Res_NEED_MORE;
UInt32 magic = GetUi32(p);
// skippable frames
if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) {
if (size < 16)
return k_IsArc_Res_NEED_MORE;
magic = GetUi32(p+12);
}
// Lz5 Magic
if (magic == 0x184D2205)
return k_IsArc_Res_YES;
return k_IsArc_Res_NO;
}
}
STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)
{
COM_TRY_BEGIN
Close();
{
Byte buf[kSignatureCheckSize];
RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize));
if (IsArc_lz5(buf, kSignatureCheckSize) == k_IsArc_Res_NO)
return S_FALSE;
_isArc = true;
_stream = stream;
_seqStream = stream;
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
}
return S_OK;
COM_TRY_END
}
STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
{
Close();
_isArc = true;
_seqStream = stream;
return S_OK;
}
STDMETHODIMP CHandler::Close()
{
_isArc = false;
_dataAfterEnd = false;
_needMoreInput = false;
_packSize_Defined = false;
_unpackSize_Defined = false;
_packSize = 0;
_seqStream.Release();
_stream.Release();
return S_OK;
}
STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
Int32 testMode, IArchiveExtractCallback *extractCallback)
{
COM_TRY_BEGIN
if (numItems == 0)
return S_OK;
if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
return E_INVALIDARG;
if (_packSize_Defined)
extractCallback->SetTotal(_packSize);
CMyComPtr<ISequentialOutStream> realOutStream;
Int32 askMode = testMode ?
NExtract::NAskMode::kTest :
NExtract::NAskMode::kExtract;
RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
if (!testMode && !realOutStream)
return S_OK;
extractCallback->PrepareOperation(askMode);
Int32 opRes;
{
NCompress::NLZ5::CDecoder *decoderSpec = new NCompress::NLZ5::CDecoder;
CMyComPtr<ICompressCoder> decoder = decoderSpec;
decoderSpec->SetInStream(_seqStream);
CDummyOutStream *outStreamSpec = new CDummyOutStream;
CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
outStreamSpec->SetStream(realOutStream);
outStreamSpec->Init();
realOutStream.Release();
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(extractCallback, true);
UInt64 packSize = 0;
UInt64 unpackedSize = 0;
HRESULT result = S_OK;
for (;;)
{
lps->InSize = packSize;
lps->OutSize = unpackedSize;
RINOK(lps->SetCur());
result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
UInt64 streamSize = decoderSpec->GetInputProcessedSize();
if (result != S_FALSE && result != S_OK)
return result;
if (unpackedSize == 0)
break;
if (streamSize == packSize)
{
// no new bytes in input stream, So it's good end of archive.
result = S_OK;
break;
}
if (packSize > streamSize)
return E_FAIL;
if (result != S_OK)
break;
}
decoderSpec->ReleaseInStream();
outStream.Release();
if (!_isArc)
opRes = NExtract::NOperationResult::kIsNotArc;
else if (_needMoreInput)
opRes = NExtract::NOperationResult::kUnexpectedEnd;
else if (_dataAfterEnd)
opRes = NExtract::NOperationResult::kDataAfterEnd;
else if (result == S_FALSE)
opRes = NExtract::NOperationResult::kDataError;
else if (result == S_OK)
opRes = NExtract::NOperationResult::kOK;
else
return result;
}
return extractCallback->SetOperationResult(opRes);
COM_TRY_END
}
static HRESULT UpdateArchive(
UInt64 unpackSize,
ISequentialOutStream *outStream,
const CProps &props,
IArchiveUpdateCallback *updateCallback)
{
RINOK(updateCallback->SetTotal(unpackSize));
CMyComPtr<ISequentialInStream> fileInStream;
RINOK(updateCallback->GetStream(0, &fileInStream));
CLocalProgress *localProgressSpec = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
localProgressSpec->Init(updateCallback, true);
NCompress::NLZ5::CEncoder *encoderSpec = new NCompress::NLZ5::CEncoder;
CMyComPtr<ICompressCoder> encoder = encoderSpec;
RINOK(props.SetCoderProps(encoderSpec, NULL));
RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));
return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
}
STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
{
*type = NFileTimeType::kUnix;
return S_OK;
}
STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
IArchiveUpdateCallback *updateCallback)
{
COM_TRY_BEGIN
if (numItems != 1)
return E_INVALIDARG;
Int32 newData, newProps;
UInt32 indexInArchive;
if (!updateCallback)
return E_FAIL;
RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive));
if ((newProps))
{
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop));
if (prop.vt != VT_EMPTY)
if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
return E_INVALIDARG;
}
}
if ((newData))
{
UInt64 size;
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
if (prop.vt != VT_UI8)
return E_INVALIDARG;
size = prop.uhVal.QuadPart;
}
return UpdateArchive(size, outStream, _props, updateCallback);
}
if (indexInArchive != 0)
return E_INVALIDARG;
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(updateCallback, true);
CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
if (opCallback)
{
RINOK(opCallback->ReportOperation(
NEventIndexType::kInArcIndex, 0,
NUpdateNotifyOp::kReplicate))
}
if (_stream)
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
return NCompress::CopyStream(_stream, outStream, progress);
COM_TRY_END
}
STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
{
return _props.SetProperties(names, values, numProps);
}
static const Byte k_Signature[] = "0x184D2205";
REGISTER_ARC_IO(
"lz5", "lz5 tlz5", "* .tar", 0x10,
k_Signature,
0,
NArcInfoFlags::kKeepName,
IsArc_lz5)
}}

460
CPP/7zip/Archive/LzHandler.cpp Normal file
View File

@@ -0,0 +1,460 @@
// LzHandler.cpp
#include "StdAfx.h"
#include "../../../C/CpuArch.h"
#include "../../Common/ComTry.h"
#include "../../Common/IntToString.h"
#include "../../Windows/PropVariant.h"
#include "../Common/ProgressUtils.h"
#include "../Common/RegisterArc.h"
#include "../Common/StreamUtils.h"
#include "../Compress/LzmaDecoder.h"
#include "Common/OutStreamWithCRC.h"
using namespace NWindows;
namespace NArchive {
namespace NLz {
static const Byte kProps[] =
{
kpidSize,
kpidPackSize,
};
static const Byte kArcProps[] =
{
kpidNumStreams
};
static const Byte k_Signature[5] = { 'L', 'Z', 'I', 'P', 1 };
enum { k_SignatureSize = 5 };
struct CHeader
{
Byte data[6]; // 0-3 magic bytes, 4 version, 5 coded_dict_size
enum { size = 6 };
enum { min_dictionary_size = 1 << 12, max_dictionary_size = 1 << 29 };
unsigned DicSize;
Byte LzmaProps[5];
bool Parse();
};
struct CTrailer
{
Byte data[20]; // 0-3 CRC32 of the uncompressed data
// 4-11 size of the uncompressed data
// 12-19 member size including header and trailer
enum { size = 20 };
unsigned data_crc() const
{
unsigned tmp = 0;
for( int i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; }
return tmp;
}
UInt64 data_size() const
{
UInt64 tmp = 0;
for( int i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; }
return tmp;
}
UInt64 member_size() const
{
UInt64 tmp = 0;
for( int i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; }
return tmp;
}
};
class CDecoder
{
CMyComPtr<ICompressCoder> _lzmaDecoder;
public:
NCompress::NLzma::CDecoder *_lzmaDecoderSpec;
~CDecoder();
HRESULT Create(ISequentialInStream *inStream);
HRESULT Code(const CHeader &header, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
UInt64 GetInputProcessedSize() const { return _lzmaDecoderSpec->GetInputProcessedSize(); }
void ReleaseInStream() { if (_lzmaDecoder) _lzmaDecoderSpec->ReleaseInStream(); }
HRESULT ReadInput(Byte *data, UInt32 size, UInt32 *processedSize)
{ return _lzmaDecoderSpec->ReadFromInputStream(data, size, processedSize); }
};
HRESULT CDecoder::Create(ISequentialInStream *inStream)
{
if (!_lzmaDecoder)
{
_lzmaDecoderSpec = new NCompress::NLzma::CDecoder;
_lzmaDecoderSpec->FinishStream = true;
_lzmaDecoder = _lzmaDecoderSpec;
}
return _lzmaDecoderSpec->SetInStream(inStream);
}
CDecoder::~CDecoder()
{
ReleaseInStream();
}
HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream,
ICompressProgressInfo *progress)
{
{
CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties;
_lzmaDecoder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecoderProperties);
if (!setDecoderProperties)
return E_NOTIMPL;
RINOK(setDecoderProperties->SetDecoderProperties2(header.LzmaProps, 5));
}
RINOK(_lzmaDecoderSpec->CodeResume(outStream, NULL, progress));
return S_OK;
}
class CHandler:
public IInArchive,
public IArchiveOpenSeq,
public CMyUnknownImp
{
CHeader _header;
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
bool _isArc;
bool _needSeekToStart;
bool _dataAfterEnd;
bool _needMoreInput;
bool _packSize_Defined;
bool _unpackSize_Defined;
bool _numStreams_Defined;
bool _dataError;
UInt64 _packSize;
UInt64 _unpackSize;
UInt64 _numStreams;
public:
MY_UNKNOWN_IMP2(IInArchive, IArchiveOpenSeq)
INTERFACE_IInArchive(;)
STDMETHOD(OpenSeq)(ISequentialInStream *stream);
CHandler() { }
};
IMP_IInArchive_Props
IMP_IInArchive_ArcProps
STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
{
NCOM::CPropVariant prop;
switch (propID)
{
case kpidPhySize: if (_packSize_Defined) prop = _packSize; break;
case kpidUnpackSize: if (_unpackSize_Defined) prop = _unpackSize; break;
case kpidNumStreams: if (_numStreams_Defined) prop = _numStreams; break;
case kpidErrorFlags:
{
UInt32 v = 0;
if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;;
if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd;
if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd;
if (_dataError) v |= kpv_ErrorFlags_DataError;
prop = v;
}
}
prop.Detach(value);
return S_OK;
}
STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
{
*numItems = 1;
return S_OK;
}
STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
{
NCOM::CPropVariant prop;
switch (propID)
{
case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
}
prop.Detach(value);
return S_OK;
}
API_FUNC_static_IsArc IsArc_Lz(const Byte *p, size_t size)
{
if (size < k_SignatureSize)
return k_IsArc_Res_NEED_MORE;
for( int i = 0; i < k_SignatureSize; ++i )
if( p[i] != k_Signature[i] )
return k_IsArc_Res_NO;
return k_IsArc_Res_YES;
}
}
bool CHeader::Parse()
{
if (IsArc_Lz(data, k_SignatureSize) == k_IsArc_Res_NO)
return false;
DicSize = ( 1 << ( data[5] & 0x1F ) );
if( DicSize > min_dictionary_size )
DicSize -= ( DicSize / 16 ) * ( ( data[5] >> 5 ) & 7 );
LzmaProps[0] = 93; /* (45 * 2) + (9 * 0) + 3 */
unsigned ds = DicSize;
for( int i = 1; i <= 4; ++i ) { LzmaProps[i] = ds & 0xFF; ds >>= 8; }
return (DicSize >= min_dictionary_size && DicSize <= max_dictionary_size);
}
STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *)
{
Close();
RINOK(ReadStream_FALSE(inStream, _header.data, CHeader::size));
if (!_header.Parse())
return S_FALSE;
RINOK(inStream->Seek(0, STREAM_SEEK_END, &_packSize));
if (_packSize < 36)
return S_FALSE;
_isArc = true;
_stream = inStream;
_seqStream = inStream;
_needSeekToStart = true;
return S_OK;
}
STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
{
Close();
_isArc = true;
_seqStream = stream;
return S_OK;
}
STDMETHODIMP CHandler::Close()
{
_isArc = false;
_packSize_Defined = false;
_unpackSize_Defined = false;
_numStreams_Defined = false;
_dataAfterEnd = false;
_needMoreInput = false;
_dataError = false;
_packSize = 0;
_needSeekToStart = false;
_stream.Release();
_seqStream.Release();
return S_OK;
}
class CCompressProgressInfoImp:
public ICompressProgressInfo,
public CMyUnknownImp
{
CMyComPtr<IArchiveOpenCallback> Callback;
public:
UInt64 Offset;
MY_UNKNOWN_IMP1(ICompressProgressInfo)
STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize);
void Init(IArchiveOpenCallback *callback) { Callback = callback; }
};
STDMETHODIMP CCompressProgressInfoImp::SetRatioInfo(const UInt64 *inSize, const UInt64 * /* outSize */)
{
if (Callback)
{
UInt64 files = 0;
UInt64 value = Offset + *inSize;
return Callback->SetCompleted(&files, &value);
}
return S_OK;
}
STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
Int32 testMode, IArchiveExtractCallback *extractCallback)
{
COM_TRY_BEGIN
if (numItems == 0)
return S_OK;
if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
return E_INVALIDARG;
if (_packSize_Defined)
extractCallback->SetTotal(_packSize);
CMyComPtr<ISequentialOutStream> realOutStream;
Int32 askMode = testMode ?
NExtract::NAskMode::kTest :
NExtract::NAskMode::kExtract;
RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
if (!testMode && !realOutStream)
return S_OK;
extractCallback->PrepareOperation(askMode);
COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC;
CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
outStreamSpec->SetStream(realOutStream);
outStreamSpec->Init();
realOutStream.Release();
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(extractCallback, true);
if (_needSeekToStart)
{
if (!_stream)
return E_FAIL;
RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
}
else
_needSeekToStart = true;
CDecoder decoder;
HRESULT result = decoder.Create(_seqStream);
RINOK(result);
bool firstItem = true;
UInt64 packSize = 0;
UInt64 unpackSize = 0;
UInt64 numStreams = 0;
bool crcError = false;
bool dataAfterEnd = false;
for (;;)
{
lps->InSize = packSize;
lps->OutSize = unpackSize;
RINOK(lps->SetCur());
CHeader st;
UInt32 processed;
RINOK(decoder.ReadInput(st.data, CHeader::size, &processed));
if (processed != CHeader::size)
{
if (processed != 0)
dataAfterEnd = true;
break;
}
if (!st.Parse())
{
dataAfterEnd = true;
break;
}
numStreams++;
firstItem = false;
outStreamSpec->InitCRC();
result = decoder.Code(st, outStream, progress);
UInt64 member_size = decoder.GetInputProcessedSize() - packSize;
packSize += member_size;
UInt64 data_size = outStreamSpec->GetSize() - unpackSize;
unpackSize += data_size;
if (result == S_OK)
{
CTrailer trailer;
RINOK(decoder.ReadInput(trailer.data, CTrailer::size, &processed));
packSize += processed; member_size += processed;
if (processed != CTrailer::size ||
trailer.data_crc() != outStreamSpec->GetCRC() ||
trailer.data_size() != data_size ||
trailer.member_size() != member_size)
{
crcError = true;
result = S_FALSE;
break;
}
}
if (result == S_FALSE)
break;
RINOK(result);
}
if (firstItem)
{
_isArc = false;
result = S_FALSE;
}
else if (result == S_OK || result == S_FALSE)
{
if (dataAfterEnd)
_dataAfterEnd = true;
else if (decoder._lzmaDecoderSpec->NeedMoreInput)
_needMoreInput = true;
_packSize = packSize;
_unpackSize = unpackSize;
_numStreams = numStreams;
_packSize_Defined = true;
_unpackSize_Defined = true;
_numStreams_Defined = true;
}
Int32 opResult = NExtract::NOperationResult::kOK;
if (!_isArc)
opResult = NExtract::NOperationResult::kIsNotArc;
else if (_needMoreInput)
opResult = NExtract::NOperationResult::kUnexpectedEnd;
else if (crcError)
opResult = NExtract::NOperationResult::kCRCError;
else if (_dataAfterEnd)
opResult = NExtract::NOperationResult::kDataAfterEnd;
else if (result == S_FALSE)
opResult = NExtract::NOperationResult::kDataError;
else if (result == S_OK)
opResult = NExtract::NOperationResult::kOK;
else
return result;
outStream.Release();
return extractCallback->SetOperationResult(opResult);
COM_TRY_END
}
REGISTER_ARC_I(
"lzip", "lz tlz", "* .tar", 0xC6,
k_Signature,
0,
NArcInfoFlags::kKeepName,
IsArc_Lz)
}}

23
CPP/7zip/Archive/ZstdHandler.cpp
View File

@@ -17,8 +17,6 @@
#include "Common/DummyOutStream.h"
#include "Common/HandlerOut.h"
#include <stdio.h>
using namespace NWindows;
namespace NArchive {
@@ -109,6 +107,14 @@ API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size)
UInt32 magic = GetUi32(p);
// skippable frames
if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) {
if (size < 16)
return k_IsArc_Res_NEED_MORE;
magic = GetUi32(p+12);
}
#ifdef ZSTD_LEGACY_SUPPORT
// zstd 0.1
if (magic == 0xFD2FB51E)
return k_IsArc_Res_YES;
@@ -116,10 +122,11 @@ API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size)
// zstd magic's for 0.2 .. 0.8 (aka 1.x)
if (magic >= 0xFD2FB522 && magic <= 0xFD2FB528)
return k_IsArc_Res_YES;
// skippable frames
if (magic >= 0x184D2A50 && magic <= 0x184D2A5F)
#else
/* only version 1.x */
if (magic == 0xFD2FB528)
return k_IsArc_Res_YES;
#endif
return k_IsArc_Res_NO;
}
@@ -223,10 +230,6 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
UInt64 streamSize = decoderSpec->GetInputProcessedSize();
printf("streamsize=%d packsize=%d unpackedSize=%d\n",
streamSize, packSize, unpackedSize);
fflush(stdout);
if (result != S_FALSE && result != S_OK)
return result;
@@ -362,7 +365,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR
return _props.SetProperties(names, values, numProps);
}
static const Byte k_Signature[] = "0xFD2FB525..0xFD2FB528";
static const Byte k_Signature[] = "0xFD2FB522..28";
REGISTER_ARC_IO(
"zstd", "zst tzstd", "* .tar", 0x0e,

4
CPP/7zip/Bundles/Alone/Alone.dsp
View File

@@ -1618,6 +1618,10 @@ SOURCE=..\..\Archive\IArchive.h
# End Source File
# Begin Source File
SOURCE=..\..\Archive\LzHandler.cpp
# End Source File
# Begin Source File
SOURCE=..\..\Archive\LzmaHandler.cpp
# End Source File
# Begin Source File

1
CPP/7zip/Bundles/Alone/makefile
View File

@@ -70,6 +70,7 @@ AR_OBJS = \
$O\Bz2Handler.obj \
$O\DeflateProps.obj \
$O\GzHandler.obj \
$O\LzHandler.obj \
$O\LzmaHandler.obj \
$O\SplitHandler.obj \
$O\XzHandler.obj \

3
CPP/7zip/Bundles/Codec_lz4/StdAfx.cpp Normal file
View File

@@ -0,0 +1,3 @@
// StdAfx.cpp
#include "StdAfx.h"

8
CPP/7zip/Bundles/Codec_lz4/StdAfx.h Normal file
View File

@@ -0,0 +1,8 @@
// StdAfx.h
#ifndef __STDAFX_H
#define __STDAFX_H
#include "../../../Common/Common.h"
#endif

36
CPP/7zip/Bundles/Codec_lz4/makefile Normal file
View File

@@ -0,0 +1,36 @@
PROG = lz4.dll
DEF_FILE = ../../Compress/Codec.def
CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID
7ZIP_COMMON_OBJS = \
$O\StreamUtils.obj \
WIN_OBJS = \
$O\System.obj \
COMPRESS_OBJS = \
$O\CodecExports.obj \
$O\DllExportsCompress.obj \
C_OBJS = \
$O\Alloc.obj \
$O\Threads.obj \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\Lz4Decoder.obj \
$O\Lz4Encoder.obj \
$O\Lz4Register.obj \
LZ4_OBJS = \
$O\lz4.obj \
$O\lz4frame.obj \
$O\lz4hc.obj \
$O\lz4xxhash.obj \
ZSTDMT_OBJS = \
$O\lz4mt_common.obj \
$O\lz4mt_compress.obj \
$O\lz4mt_decompress.obj \
$O\threading.obj \
!include "../../7zip.mak"

6
CPP/7zip/Bundles/Codec_lz4/resource.rc Normal file
View File

@@ -0,0 +1,6 @@
#include "../../../../C/7zVersionTr.h"
#include "../../../../C/7zVersion.rc"
MY_VERSION_INFO_DLL("7z LZ4 Plugin", "lz4")
101 ICON "../../Archive/Icons/7z.ico"

3
CPP/7zip/Bundles/Codec_lz5/StdAfx.cpp Normal file
View File

@@ -0,0 +1,3 @@
// StdAfx.cpp
#include "StdAfx.h"

8
CPP/7zip/Bundles/Codec_lz5/StdAfx.h Normal file
View File

@@ -0,0 +1,8 @@
// StdAfx.h
#ifndef __STDAFX_H
#define __STDAFX_H
#include "../../../Common/Common.h"
#endif

36
CPP/7zip/Bundles/Codec_lz5/makefile Normal file
View File

@@ -0,0 +1,36 @@
PROG = lz5.dll
DEF_FILE = ../../Compress/Codec.def
CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID
7ZIP_COMMON_OBJS = \
$O\StreamUtils.obj \
WIN_OBJS = \
$O\System.obj \
COMPRESS_OBJS = \
$O\CodecExports.obj \
$O\DllExportsCompress.obj \
C_OBJS = \
$O\Alloc.obj \
$O\Threads.obj \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\Lz5Decoder.obj \
$O\Lz5Encoder.obj \
$O\Lz5Register.obj \
LZ5_OBJS = \
$O\lz5.obj \
$O\lz5frame.obj \
$O\lz5hc.obj \
$O\lz5xxhash.obj \
ZSTDMT_OBJS = \
$O\lz5mt_common.obj \
$O\lz5mt_compress.obj \
$O\lz5mt_decompress.obj \
$O\threading.obj \
!include "../../7zip.mak"

6
CPP/7zip/Bundles/Codec_lz5/resource.rc Normal file
View File

@@ -0,0 +1,6 @@
#include "../../../../C/7zVersionTr.h"
#include "../../../../C/7zVersion.rc"
MY_VERSION_INFO_DLL("7z LZ5 Plugin", "lz5")
101 ICON "../../Archive/Icons/7z.ico"

25
CPP/7zip/Bundles/Codec_zstd/makefile
View File

@@ -5,17 +5,38 @@ CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID
7ZIP_COMMON_OBJS = \
$O\StreamUtils.obj \
WIN_OBJS = \
$O\System.obj \
COMPRESS_OBJS = \
$O\CodecExports.obj \
$O\DllExportsCompress.obj \
C_OBJS = \
$O\Alloc.obj \
$O\Threads.obj \
!include "../../zstd.mak"
ZSTD_OBJS = $(ZSTD_OBJS) \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\ZstdDecoder.obj \
$O\ZstdEncoder.obj \
$O\ZstdRegister.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\fse_compress.obj \
$O\huf_compress.obj \
$O\zstd_compress.obj \
$O\error_private.obj \
ZSTDMT_OBJS = \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_compress.obj \
$O\zstdmt_decompress.obj \
!include "../../7zip.mak"

25
CPP/7zip/Bundles/Codec_zstdF/makefile
View File

@@ -5,18 +5,33 @@ CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_LEGACY_SUPPORT
7ZIP_COMMON_OBJS = \
$O\StreamUtils.obj \
WIN_OBJS = \
$O\System.obj \
COMPRESS_OBJS = \
$O\CodecExports.obj \
$O\DllExportsCompress.obj \
C_OBJS = \
$O\Alloc.obj \
$O\Threads.obj \
!include "../../zstd.mak"
ZSTD_OBJS = $(ZSTD_OBJS) \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\ZstdDecoder.obj \
$O\ZstdEncoder.obj \
$O\ZstdRegister.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\fse_compress.obj \
$O\huf_compress.obj \
$O\zstd_compress.obj \
$O\error_private.obj \
$O\zstd_v01.obj \
$O\zstd_v02.obj \
$O\zstd_v03.obj \
@@ -25,4 +40,10 @@ ZSTD_OBJS = $(ZSTD_OBJS) \
$O\zstd_v06.obj \
$O\zstd_v07.obj \
ZSTDMT_OBJS = \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_compress.obj \
$O\zstdmt_decompress.obj \
!include "../../7zip.mak"

2
CPP/7zip/Bundles/Fm/resource.rc
View File

@@ -3,5 +3,5 @@
STRINGTABLE
BEGIN
100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs"
100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lz tlz lz4 tlz4 lz5 tlz5 lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs zst"
END

22
CPP/7zip/Bundles/Format7z/makefile
View File

@@ -136,10 +136,28 @@ C_OBJS = \
!include "../../Aes.mak"
!include "../../Crc.mak"
!include "../../zstd.mak"
ZSTD_OBJS = $(ZSTD_OBJS) \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\ZstdDecoder.obj \
$O\ZstdEncoder.obj \
$O\ZstdRegister.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\fse_compress.obj \
$O\huf_compress.obj \
$O\zstd_compress.obj \
$O\error_private.obj \
ZSTDMT_OBJS = \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_compress.obj \
$O\zstdmt_decompress.obj \
!include "../../7zip.mak"

5
CPP/7zip/Bundles/Format7zF/Arc.mak
View File

@@ -69,6 +69,9 @@ AR_OBJS = \
$O\HandlerCont.obj \
$O\HfsHandler.obj \
$O\IhexHandler.obj \
$O\LzHandler.obj \
$O\Lz4Handler.obj \
$O\Lz5Handler.obj \
$O\LzhHandler.obj \
$O\LzmaHandler.obj \
$O\MachoHandler.obj \
@@ -104,7 +107,6 @@ AR_COMMON_OBJS = \
$O\HandlerOut.obj \
$O\ParseProperties.obj \
7Z_OBJS = \
$O\7zCompressionMode.obj \
$O\7zDecode.obj \
@@ -242,7 +244,6 @@ CRYPTO_OBJS = \
$O\ZipCrypto.obj \
$O\ZipStrong.obj \
C_OBJS = \
$O\7zBuf2.obj \
$O\7zStream.obj \

43
CPP/7zip/Bundles/Format7zF/makefile
View File

@@ -16,12 +16,39 @@ AR_OBJS = $(AR_OBJS) \
$O\ArchiveExports.obj \
$O\DllExports2.obj \
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\Lz4Decoder.obj \
$O\Lz4Encoder.obj \
$O\Lz4Register.obj \
$O\Lz5Decoder.obj \
$O\Lz5Encoder.obj \
$O\Lz5Register.obj \
$O\ZstdDecoder.obj \
$O\ZstdEncoder.obj \
$O\ZstdRegister.obj \
!include "../../zstd.mak"
ZSTD_OBJS = $(ZSTD_OBJS) \
LZ4_OBJS = \
$O\lz4.obj \
$O\lz4frame.obj \
$O\lz4hc.obj \
$O\lz4xxhash.obj \
LZ5_OBJS = \
$O\lz5.obj \
$O\lz5frame.obj \
$O\lz5hc.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\fse_compress.obj \
$O\huf_compress.obj \
$O\zstd_compress.obj \
$O\error_private.obj \
$O\zstd_v01.obj \
$O\zstd_v02.obj \
$O\zstd_v03.obj \
@@ -30,4 +57,16 @@ ZSTD_OBJS = $(ZSTD_OBJS) \
$O\zstd_v06.obj \
$O\zstd_v07.obj \
ZSTDMT_OBJS = \
$O\lz5mt_common.obj \
$O\lz5mt_compress.obj \
$O\lz5mt_decompress.obj \
$O\lz4mt_common.obj \
$O\lz4mt_compress.obj \
$O\lz4mt_decompress.obj \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_compress.obj \
$O\zstdmt_decompress.obj \
!include "../../7zip.mak"

4
CPP/7zip/Bundles/Format7zF/resource.rc
View File

@@ -29,9 +29,11 @@ MY_VERSION_INFO_DLL("7z Plugin", "7z")
23 ICON "../../Archive/Icons/xz.ico"
24 ICON "../../Archive/Icons/squashfs.ico"
25 ICON "../../Archive/Icons/zst.ico"
26 ICON "../../Archive/Icons/lz4.ico"
27 ICON "../../Archive/Icons/lz5.ico"
STRINGTABLE
BEGIN
100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25"
100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lz:16 tlz:16 lz4:26 lz5:27 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25"
END

20
CPP/7zip/Bundles/SFXCon/makefile
View File

@@ -41,6 +41,7 @@ WIN_OBJS = \
$O\PropVariant.obj \
$O\PropVariantConv.obj \
$O\Synchronization.obj \
$O\System.obj \
7ZIP_COMMON_OBJS = \
$O\CreateCoder.obj \
@@ -126,5 +127,22 @@ C_OBJS = \
!include "../../Aes.mak"
!include "../../Crc.mak"
!include "../../zstd.mak"
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\ZstdDecoder.obj \
$O\ZstdRegister.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\error_private.obj \
ZSTDMT_OBJS = \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_decompress.obj \
!include "../../7zip.mak"

2
CPP/7zip/Bundles/SFXSetup/makefile
View File

@@ -106,6 +106,4 @@ C_OBJS = \
$O\Threads.obj \
!include "../../Crc.mak"
!include "../../zstd.mak"
!include "../../7zip.mak"

20
CPP/7zip/Bundles/SFXWin/makefile
View File

@@ -40,6 +40,7 @@ WIN_OBJS = \
$O\PropVariantConv.obj \
$O\ResourceString.obj \
$O\Shell.obj \
$O\System.obj \
$O\Synchronization.obj \
$O\Window.obj \
@@ -144,5 +145,22 @@ C_OBJS = \
!include "../../Aes.mak"
!include "../../Crc.mak"
!include "../../zstd.mak"
COMPRESS_OBJS = $(COMPRESS_OBJS) \
$O\ZstdDecoder.obj \
$O\ZstdRegister.obj \
ZSTD_OBJS = \
$O\entropy_common.obj \
$O\fse_decompress.obj \
$O\huf_decompress.obj \
$O\zstd_common.obj \
$O\zstd_decompress.obj \
$O\xxhash.obj \
$O\error_private.obj \
ZSTDMT_OBJS = \
$O\threading.obj \
$O\zstdmt_common.obj \
$O\zstdmt_decompress.obj \
!include "../../7zip.mak"

177
CPP/7zip/Compress/Lz4Decoder.cpp Normal file
View File

@@ -0,0 +1,177 @@
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "Lz4Decoder.h"
int Lz4Read(void *arg, LZ4MT_Buffer * in)
{
struct Lz4Stream *x = (struct Lz4Stream*)arg;
size_t size = in->size;
HRESULT res = ReadStream(x->inStream, in->buf, &size);
if (res != S_OK)
return -1;
in->size = size;
*x->processedIn += size;
return 0;
}
int Lz4Write(void *arg, LZ4MT_Buffer * out)
{
struct Lz4Stream *x = (struct Lz4Stream*)arg;
UInt32 todo = (UInt32)out->size;
UInt32 done = 0;
while (todo != 0)
{
UInt32 block;
HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block);
done += block;
if (res == k_My_HRESULT_WritingWasCut)
break;
if (res != S_OK)
return -1;
if (block == 0)
return E_FAIL;
todo -= block;
}
*x->processedOut += done;
if (x->progress)
x->progress->SetRatioInfo(x->processedIn, x->processedOut);
return 0;
}
namespace NCompress {
namespace NLZ4 {
CDecoder::CDecoder():
_processedIn(0),
_processedOut(0),
_inputSize(0),
_numThreads(NWindows::NSystem::GetNumberOfProcessors())
{
_props.clear();
}
CDecoder::~CDecoder()
{
}
HRESULT CDecoder::ErrorOut(size_t code)
{
const char *strError = LZ4MT_getErrorString(code);
wchar_t wstrError[200+5]; /* no malloc here, /TR */
mbstowcs(wstrError, strError, 200);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return S_FALSE;
}
STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
{
DProps *pProps = (DProps *)prop;
if (size != sizeof(DProps))
return E_FAIL;
memcpy(&_props, pProps, sizeof (DProps));
return S_OK;
}
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
{
const UInt32 kNumThreadsMax = LZ4MT_THREAD_MAX;
if (numThreads < 1) numThreads = 1;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads;
return S_OK;
}
HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
{
_processedOut = 0;
return S_OK;
}
STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
{
_processedIn = 0;
RINOK(SetOutStreamSizeResume(outSize));
return S_OK;
}
HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream,
ISequentialOutStream * outStream, ICompressProgressInfo * progress)
{
LZ4MT_RdWr_t rdwr;
size_t result;
HRESULT res = S_OK;
struct Lz4Stream Rd;
Rd.inStream = inStream;
Rd.processedIn = &_processedIn;
struct Lz4Stream Wr;
Wr.progress = progress;
Wr.outStream = outStream;
Wr.processedIn = &_processedIn;
Wr.processedOut = &_processedOut;
/* 1) setup read/write functions */
rdwr.fn_read = ::Lz4Read;
rdwr.fn_write = ::Lz4Write;
rdwr.arg_read = (void *)&Rd;
rdwr.arg_write = (void *)&Wr;
/* 2) create compression context */
LZ4MT_DCtx *ctx = LZ4MT_createDCtx(_numThreads, _inputSize);
if (!ctx)
return S_FALSE;
/* 3) compress */
result = LZ4MT_DecompressDCtx(ctx, &rdwr);
if (result == (size_t)-LZ4MT_error_read_fail)
res = E_ABORT;
else if (LZ4MT_isError(result))
return ErrorOut(result);
/* 4) free resources */
LZ4MT_freeDCtx(ctx);
return res;
}
STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress)
{
SetOutStreamSize(outSize);
return CodeSpec(inStream, outStream, progress);
}
#ifndef NO_READ_FROM_CODER
STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
{
_inStream = inStream;
return S_OK;
}
STDMETHODIMP CDecoder::ReleaseInStream()
{
_inStream.Release();
return S_OK;
}
#endif
HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
{
RINOK(SetOutStreamSizeResume(outSize));
return CodeSpec(_inStream, outStream, progress);
}
}}

99
CPP/7zip/Compress/Lz4Decoder.h Normal file
View File

@@ -0,0 +1,99 @@
// (C) 2016 Tino Reichardt
/**
* you can define LZ4_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7)
* /TR 2016-10-01
*/
#define LZ4_STATIC_LINKING_ONLY
#include "../../../C/Alloc.h"
#include "../../../C/Threads.h"
#include "../../../C/lz4/lz4.h"
#include "../../../C/zstdmt/lz4mt.h"
#include "../../Windows/System.h"
#include "../../Common/Common.h"
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../Common/StreamUtils.h"
#include "../Common/RegisterCodec.h"
#include "../Common/ProgressMt.h"
struct Lz4Stream {
ISequentialInStream *inStream;
ISequentialOutStream *outStream;
ICompressProgressInfo *progress;
UInt64 *processedIn;
UInt64 *processedOut;
CCriticalSection *cs;
int flags;
};
extern int Lz4Read(void *Stream, LZ4MT_Buffer * in);
extern int Lz4Write(void *Stream, LZ4MT_Buffer * in);
namespace NCompress {
namespace NLZ4 {
struct DProps
{
DProps() { clear (); }
void clear ()
{
memset(this, 0, sizeof (*this));
_ver_major = LZ4_VERSION_MAJOR;
_ver_minor = LZ4_VERSION_MINOR;
_level = 1;
}
Byte _ver_major;
Byte _ver_minor;
Byte _level;
Byte _reserved[2];
};
class CDecoder:public ICompressCoder,
public ICompressSetDecoderProperties2,
public CMyUnknownImp
{
CMyComPtr < ISequentialInStream > _inStream;
DProps _props;
CCriticalSection cs;
UInt64 _processedIn;
UInt64 _processedOut;
UInt32 _inputSize;
UInt32 _numThreads;
HRESULT CDecoder::ErrorOut(size_t code);
HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
public:
MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2)
#ifndef NO_READ_FROM_CODER
MY_QUERYINTERFACE_ENTRY(ICompressSetInStream)
#endif
MY_QUERYINTERFACE_END
MY_ADDREF_RELEASE
STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size);
STDMETHOD (SetOutStreamSize)(const UInt64 *outSize);
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads);
#ifndef NO_READ_FROM_CODER
STDMETHOD (SetInStream)(ISequentialInStream *inStream);
STDMETHOD (ReleaseInStream)();
UInt64 GetInputProcessedSize() const { return _processedIn; }
#endif
HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
CDecoder();
virtual ~CDecoder();
};
}}

139
CPP/7zip/Compress/Lz4Encoder.cpp Normal file
View File

@@ -0,0 +1,139 @@
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "Lz4Encoder.h"
#include "Lz4Decoder.h"
#ifndef EXTRACT_ONLY
namespace NCompress {
namespace NLZ4 {
CEncoder::CEncoder():
_processedIn(0),
_processedOut(0),
_inputSize(0),
_ctx(NULL),
_numThreads(NWindows::NSystem::GetNumberOfProcessors())
{
_props.clear();
}
CEncoder::~CEncoder()
{
if (_ctx)
LZ4MT_freeCCtx(_ctx);
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
{
_props.clear();
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT & prop = coderProps[i];
PROPID propID = propIDs[i];
UInt32 v = (UInt32)prop.ulVal;
switch (propID)
{
case NCoderPropID::kLevel:
{
if (prop.vt != VT_UI4)
return E_INVALIDARG;
_props._level = static_cast < Byte > (prop.ulVal);
Byte lz4_level = static_cast < Byte > (LZ4MT_LEVEL_MAX);
if (_props._level > lz4_level)
_props._level = lz4_level;
break;
}
case NCoderPropID::kNumThreads:
{
SetNumberOfThreads(v);
break;
}
default:
{
break;
}
}
}
return S_OK;
}
STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream)
{
return WriteStream(outStream, &_props, sizeof (_props));
}
STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 * /*inSize*/ ,
const UInt64 * /*outSize */, ICompressProgressInfo *progress)
{
LZ4MT_RdWr_t rdwr;
size_t result;
HRESULT res = S_OK;
struct Lz4Stream Rd;
Rd.inStream = inStream;
Rd.outStream = outStream;
Rd.processedIn = &_processedIn;
Rd.processedOut = &_processedOut;
struct Lz4Stream Wr;
if (_processedIn == 0)
Wr.progress = progress;
else
Wr.progress = 0;
Wr.inStream = inStream;
Wr.outStream = outStream;
Wr.processedIn = &_processedIn;
Wr.processedOut = &_processedOut;
/* 1) setup read/write functions */
rdwr.fn_read = ::Lz4Read;
rdwr.fn_write = ::Lz4Write;
rdwr.arg_read = (void *)&Rd;
rdwr.arg_write = (void *)&Wr;
/* 2) create compression context, if needed */
if (!_ctx)
_ctx = LZ4MT_createCCtx(_numThreads, _props._level, _inputSize);
if (!_ctx)
return S_FALSE;
/* 3) compress */
result = LZ4MT_CompressCCtx(_ctx, &rdwr);
if (result == (size_t)-LZ4MT_error_read_fail)
res = E_ABORT;
else if (LZ4MT_isError(result))
if (result != LZ4MT_error_read_fail)
return ErrorOut(result);
return res;
}
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
{
const UInt32 kNumThreadsMax = LZ4MT_THREAD_MAX;
if (numThreads < 1) numThreads = 1;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads;
return S_OK;
}
HRESULT CEncoder::ErrorOut(size_t code)
{
const char *strError = LZ4MT_getErrorString(code);
wchar_t wstrError[200+5]; /* no malloc here, /TR */
mbstowcs(wstrError, strError, 200);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return S_FALSE;
}
}}
#endif

63
CPP/7zip/Compress/Lz4Encoder.h Normal file
View File

@@ -0,0 +1,63 @@
// (C) 2016 Tino Reichardt
#define LZ4_STATIC_LINKING_ONLY
#include "../../../C/Alloc.h"
#include "../../../C/Threads.h"
#include "../../../C/lz4/lz4.h"
#include "../../../C/zstdmt/lz4mt.h"
#include "../../Common/Common.h"
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../Common/StreamUtils.h"
#ifndef EXTRACT_ONLY
namespace NCompress {
namespace NLZ4 {
struct CProps
{
CProps() { clear (); }
void clear ()
{
memset(this, 0, sizeof (*this));
_ver_major = LZ4_VERSION_MAJOR;
_ver_minor = LZ4_VERSION_MINOR;
_level = 3;
}
Byte _ver_major;
Byte _ver_minor;
Byte _level;
Byte _reserved[2];
};
class CEncoder:
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public CMyUnknownImp
{
CProps _props;
UInt64 _processedIn;
UInt64 _processedOut;
UInt32 _inputSize;
UInt32 _numThreads;
LZ4MT_CCtx *_ctx;
HRESULT CEncoder::ErrorOut(size_t code);
public:
MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream);
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads);
CEncoder();
virtual ~CEncoder();
};
}}
#endif

17
CPP/7zip/Compress/Lz4Register.cpp Normal file
View File

@@ -0,0 +1,17 @@
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "../Common/RegisterCodec.h"
#include "Lz4Decoder.h"
#ifndef EXTRACT_ONLY
#include "Lz4Encoder.h"
#endif
REGISTER_CODEC_E(
LZ4,
NCompress::NLZ4::CDecoder(),
NCompress::NLZ4::CEncoder(),
0x4F71104, "LZ4")

178
CPP/7zip/Compress/Lz5Decoder.cpp Normal file
View File

@@ -0,0 +1,178 @@
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "Lz5Decoder.h"
int Lz5Read(void *arg, LZ5MT_Buffer * in)
{
struct Lz5Stream *x = (struct Lz5Stream*)arg;
size_t size = in->size;
HRESULT res = ReadStream(x->inStream, in->buf, &size);
if (res != S_OK)
return -1;
in->size = size;
*x->processedIn += size;
return 0;
}
int Lz5Write(void *arg, LZ5MT_Buffer * out)
{
struct Lz5Stream *x = (struct Lz5Stream*)arg;
UInt32 todo = (UInt32)out->size;
UInt32 done = 0;
while (todo != 0)
{
UInt32 block;
HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block);
done += block;
if (res == k_My_HRESULT_WritingWasCut)
break;
if (res != S_OK)
return -1;
if (block == 0)
return E_FAIL;
todo -= block;
}
*x->processedOut += done;
if (x->progress)
x->progress->SetRatioInfo(x->processedIn, x->processedOut);
return 0;
}
namespace NCompress {
namespace NLZ5 {
CDecoder::CDecoder():
_processedIn(0),
_processedOut(0),
_inputSize(0),
_numThreads(NWindows::NSystem::GetNumberOfProcessors())
{
_props.clear();
}
CDecoder::~CDecoder()
{
}
HRESULT CDecoder::ErrorOut(size_t code)
{
const char *strError = LZ5MT_getErrorString(code);
wchar_t wstrError[200+5]; /* no malloc here, /TR */
mbstowcs(wstrError, strError, 200);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return S_FALSE;
}
STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
{
DProps *pProps = (DProps *)prop;
if (size != sizeof(DProps))
return E_FAIL;
memcpy(&_props, pProps, sizeof (DProps));
return S_OK;
}
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
{
const UInt32 kNumThreadsMax = LZ5MT_THREAD_MAX;
if (numThreads < 1) numThreads = 1;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads;
return S_OK;
}
HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
{
_processedOut = 0;
return S_OK;
}
STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
{
_processedIn = 0;
RINOK(SetOutStreamSizeResume(outSize));
return S_OK;
}
HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream,
ISequentialOutStream * outStream, ICompressProgressInfo * progress)
{
LZ5MT_RdWr_t rdwr;
size_t result;
HRESULT res = S_OK;
struct Lz5Stream Rd;
Rd.inStream = inStream;
Rd.processedIn = &_processedIn;
struct Lz5Stream Wr;
Wr.progress = progress;
Wr.outStream = outStream;
Wr.processedIn = &_processedIn;
Wr.processedOut = &_processedOut;
/* 1) setup read/write functions */
rdwr.fn_read = ::Lz5Read;
rdwr.fn_write = ::Lz5Write;
rdwr.arg_read = (void *)&Rd;
rdwr.arg_write = (void *)&Wr;
/* 2) create compression context */
LZ5MT_DCtx *ctx = LZ5MT_createDCtx(_numThreads, _inputSize);
if (!ctx)
return S_FALSE;
/* 3) compress */
result = LZ5MT_DecompressDCtx(ctx, &rdwr);
if (result == (size_t)-LZ5MT_error_read_fail)
res = E_ABORT;
else if (LZ5MT_isError(result))
if (result != LZ5MT_error_read_fail)
return ErrorOut(result);
/* 4) free resources */
LZ5MT_freeDCtx(ctx);
return res;
}
STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress)
{
SetOutStreamSize(outSize);
return CodeSpec(inStream, outStream, progress);
}
#ifndef NO_READ_FROM_CODER
STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
{
_inStream = inStream;
return S_OK;
}
STDMETHODIMP CDecoder::ReleaseInStream()
{
_inStream.Release();
return S_OK;
}
#endif
HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
{
RINOK(SetOutStreamSizeResume(outSize));
return CodeSpec(_inStream, outStream, progress);
}
}}

99
CPP/7zip/Compress/Lz5Decoder.h Normal file
View File

@@ -0,0 +1,99 @@
// (C) 2016 Tino Reichardt
/**
* you can define LZ5_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7)
* /TR 2016-10-01
*/
#define LZ5_STATIC_LINKING_ONLY
#include "../../../C/Alloc.h"
#include "../../../C/Threads.h"
#include "../../../C/lz5/lz5.h"
#include "../../../C/zstdmt/lz5mt.h"
#include "../../Windows/System.h"
#include "../../Common/Common.h"
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../Common/StreamUtils.h"
#include "../Common/RegisterCodec.h"
#include "../Common/ProgressMt.h"
struct Lz5Stream {
ISequentialInStream *inStream;
ISequentialOutStream *outStream;
ICompressProgressInfo *progress;
UInt64 *processedIn;
UInt64 *processedOut;
CCriticalSection *cs;
int flags;
};
extern int Lz5Read(void *Stream, LZ5MT_Buffer * in);
extern int Lz5Write(void *Stream, LZ5MT_Buffer * in);
namespace NCompress {
namespace NLZ5 {
struct DProps
{
DProps() { clear (); }
void clear ()
{
memset(this, 0, sizeof (*this));
_ver_major = LZ5_VERSION_MAJOR;
_ver_minor = LZ5_VERSION_MINOR;
_level = 1;
}
Byte _ver_major;
Byte _ver_minor;
Byte _level;
Byte _reserved[2];
};
class CDecoder:public ICompressCoder,
public ICompressSetDecoderProperties2,
public CMyUnknownImp
{
CMyComPtr < ISequentialInStream > _inStream;
DProps _props;
CCriticalSection cs;
UInt64 _processedIn;
UInt64 _processedOut;
UInt32 _inputSize;
UInt32 _numThreads;
HRESULT CDecoder::ErrorOut(size_t code);
HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
public:
MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2)
#ifndef NO_READ_FROM_CODER
MY_QUERYINTERFACE_ENTRY(ICompressSetInStream)
#endif
MY_QUERYINTERFACE_END
MY_ADDREF_RELEASE
STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size);
STDMETHOD (SetOutStreamSize)(const UInt64 *outSize);
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads);
#ifndef NO_READ_FROM_CODER
STDMETHOD (SetInStream)(ISequentialInStream *inStream);
STDMETHOD (ReleaseInStream)();
UInt64 GetInputProcessedSize() const { return _processedIn; }
#endif
HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
CDecoder();
virtual ~CDecoder();
};
}}

138
CPP/7zip/Compress/Lz5Encoder.cpp Normal file
View File

@@ -0,0 +1,138 @@
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "Lz5Encoder.h"
#include "Lz5Decoder.h"
#ifndef EXTRACT_ONLY
namespace NCompress {
namespace NLZ5 {
CEncoder::CEncoder():
_processedIn(0),
_processedOut(0),
_inputSize(0),
_ctx(NULL),
_numThreads(NWindows::NSystem::GetNumberOfProcessors())
{
_props.clear();
}
CEncoder::~CEncoder()
{
if (_ctx)
LZ5MT_freeCCtx(_ctx);
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
{
_props.clear();
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT & prop = coderProps[i];
PROPID propID = propIDs[i];
UInt32 v = (UInt32)prop.ulVal;
switch (propID)
{
case NCoderPropID::kLevel:
{
if (prop.vt != VT_UI4)
return E_INVALIDARG;
_props._level = static_cast < Byte > (prop.ulVal);
Byte lz5_level = static_cast < Byte > (LZ5MT_LEVEL_MAX);
if (_props._level > lz5_level)
_props._level = lz5_level;
break;
}
case NCoderPropID::kNumThreads:
{
SetNumberOfThreads(v);
break;
}
default:
{
break;
}
}
}
return S_OK;
}
STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream)
{
return WriteStream(outStream, &_props, sizeof (_props));
}
STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 * /*inSize*/ ,
const UInt64 * /*outSize */, ICompressProgressInfo *progress)
{
LZ5MT_RdWr_t rdwr;
size_t result;
HRESULT res = S_OK;
struct Lz5Stream Rd;
Rd.inStream = inStream;
Rd.outStream = outStream;
Rd.processedIn = &_processedIn;
Rd.processedOut = &_processedOut;
struct Lz5Stream Wr;
if (_processedIn == 0)
Wr.progress = progress;
else
Wr.progress = 0;
Wr.inStream = inStream;
Wr.outStream = outStream;
Wr.processedIn = &_processedIn;
Wr.processedOut = &_processedOut;
/* 1) setup read/write functions */
rdwr.fn_read = ::Lz5Read;
rdwr.fn_write = ::Lz5Write;
rdwr.arg_read = (void *)&Rd;
rdwr.arg_write = (void *)&Wr;
/* 2) create compression context, if needed */
if (!_ctx)
_ctx = LZ5MT_createCCtx(_numThreads, _props._level, _inputSize);
if (!_ctx)
return S_FALSE;
/* 3) compress */
result = LZ5MT_CompressCCtx(_ctx, &rdwr);
if (result == (size_t)-LZ5MT_error_read_fail)
res = E_ABORT;
if (LZ5MT_isError(result))
return ErrorOut(result);
return res;
}
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
{
const UInt32 kNumThreadsMax = LZ5MT_THREAD_MAX;
if (numThreads < 1) numThreads = 1;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads;
return S_OK;
}
HRESULT CEncoder::ErrorOut(size_t code)
{
const char *strError = LZ5MT_getErrorString(code);
wchar_t wstrError[200+5]; /* no malloc here, /TR */
mbstowcs(wstrError, strError, 200);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return S_FALSE;
}
}}
#endif

63
CPP/7zip/Compress/Lz5Encoder.h Normal file
View File

@@ -0,0 +1,63 @@
// (C) 2016 Tino Reichardt
#define LZ5_STATIC_LINKING_ONLY
#include "../../../C/Alloc.h"
#include "../../../C/Threads.h"
#include "../../../C/lz5/lz5.h"
#include "../../../C/zstdmt/lz5mt.h"
#include "../../Common/Common.h"
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../Common/StreamUtils.h"
#ifndef EXTRACT_ONLY
namespace NCompress {
namespace NLZ5 {
struct CProps
{
CProps() { clear (); }
void clear ()
{
memset(this, 0, sizeof (*this));
_ver_major = LZ5_VERSION_MAJOR;
_ver_minor = LZ5_VERSION_MINOR;
_level = 3;
}
Byte _ver_major;
Byte _ver_minor;
Byte _level;
Byte _reserved[2];
};
class CEncoder:
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public CMyUnknownImp
{
CProps _props;
UInt64 _processedIn;
UInt64 _processedOut;
UInt32 _inputSize;
UInt32 _numThreads;
LZ5MT_CCtx *_ctx;
HRESULT CEncoder::ErrorOut(size_t code);
public:
MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream);
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads);
CEncoder();
virtual ~CEncoder();
};
}}
#endif

17
CPP/7zip/Compress/Lz5Register.cpp Normal file
View File

@@ -0,0 +1,17 @@
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "../Common/RegisterCodec.h"
#include "Lz5Decoder.h"
#ifndef EXTRACT_ONLY
#include "Lz5Encoder.h"
#endif
REGISTER_CODEC_E(
LZ5,
NCompress::NLZ5::CDecoder(),
NCompress::NLZ5::CEncoder(),
0x4F71105, "LZ5")

299
CPP/7zip/Compress/ZstdDecoder.cpp
View File

@@ -1,33 +1,76 @@
// ZstdDecoder.cpp
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "ZstdDecoder.h"
int ZstdRead(void *arg, ZSTDMT_Buffer * in)
{
struct ZstdStream *x = (struct ZstdStream*)arg;
size_t size = in->size;
HRESULT res = ReadStream(x->inStream, in->buf, &size);
if (res != S_OK)
return -1;
in->size = size;
*x->processedIn += size;
return 0;
}
int ZstdWrite(void *arg, ZSTDMT_Buffer * out)
{
struct ZstdStream *x = (struct ZstdStream*)arg;
UInt32 todo = (UInt32)out->size;
UInt32 done = 0;
while (todo != 0)
{
UInt32 block;
HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block);
done += block;
if (res == k_My_HRESULT_WritingWasCut)
break;
if (res != S_OK)
return -1;
if (block == 0)
return E_FAIL;
todo -= block;
}
*x->processedOut += done;
if (x->progress)
x->progress->SetRatioInfo(x->processedIn, x->processedOut);
return 0;
}
namespace NCompress {
namespace NZSTD {
CDecoder::CDecoder():
_dstream(NULL),
_buffIn(NULL),
_buffOut(NULL),
_buffInSizeAllocated(0),
_buffOutSizeAllocated(0),
_buffInSize(ZSTD_DStreamInSize()),
_buffOutSize(ZSTD_DStreamOutSize()*4),
_processedIn(0),
_processedOut(0)
_processedOut(0),
_inputSize(0),
_numThreads(NWindows::NSystem::GetNumberOfProcessors())
{
_props.clear();
}
CDecoder::~CDecoder()
{
if (_dstream)
ZSTD_freeDStream(_dstream);
}
MyFree(_buffIn);
MyFree(_buffOut);
HRESULT CDecoder::ErrorOut(size_t code)
{
const char *strError = ZSTDMT_getErrorString(code);
wchar_t wstrError[200+5]; /* no malloc here, /TR */
mbstowcs(wstrError, strError, 200);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return S_FALSE;
}
STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
@@ -37,97 +80,23 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
if (size != sizeof(DProps))
return E_FAIL;
#ifdef ZSTD_LEGACY_SUPPORT
/* version 0.x and 1.x are okay */
if (pProps->_ver_major > 1)
return E_FAIL;
/* 0.5, 0.6, 0.7, 0.8 are supported! */
if (pProps->_ver_major == 0) {
switch (pProps->_ver_minor) {
case 5:
break;
case 6:
break;
case 7:
break;
case 8:
break;
default:
return E_FAIL;
}}
#else
/* only exact version is okay */
if (pProps->_ver_major != ZSTD_VERSION_MAJOR)
return E_FAIL;
if (pProps->_ver_minor != ZSTD_VERSION_MINOR)
return E_FAIL;
#endif
memcpy(&_props, pProps, sizeof (DProps));
return S_OK;
}
HRESULT CDecoder::ErrorOut(size_t code)
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
{
const char *strError = ZSTD_getErrorName(code);
size_t strErrorLen = strlen(strError) + 1;
wchar_t *wstrError = (wchar_t *)MyAlloc(sizeof(wchar_t) * strErrorLen);
if (!wstrError)
return E_FAIL;
mbstowcs(wstrError, strError, strErrorLen - 1);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return E_FAIL;
}
HRESULT CDecoder::CreateDecompressor()
{
size_t result;
if (!_dstream) {
_dstream = ZSTD_createDStream();
if (!_dstream)
return E_FAIL;
}
result = ZSTD_initDStream(_dstream);
if (ZSTD_isError(result))
return ErrorOut(result);
/* allocate buffers */
if (_buffInSizeAllocated != _buffInSize)
{
MyFree(_buffIn);
_buffIn = MyAlloc(_buffInSize);
if (!_buffIn)
return E_OUTOFMEMORY;
_buffInSizeAllocated = _buffInSize;
}
if (_buffOutSizeAllocated != _buffOutSize)
{
MyFree(_buffOut);
_buffOut = MyAlloc(_buffOutSize);
if (!_buffOut)
return E_OUTOFMEMORY;
_buffOutSizeAllocated = _buffOutSize;
}
const UInt32 kNumThreadsMax = ZSTDMT_THREAD_MAX;
if (numThreads < 1) numThreads = 1;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads;
return S_OK;
}
HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
{
_processedOut = 0;
RINOK(CreateDecompressor());
return S_OK;
}
@@ -135,73 +104,48 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
{
_processedIn = 0;
RINOK(SetOutStreamSizeResume(outSize));
return S_OK;
}
STDMETHODIMP CDecoder::SetInBufSize(UInt32, UInt32 size)
HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream,
ISequentialOutStream * outStream, ICompressProgressInfo * progress)
{
_buffInSize = size;
return S_OK;
}
STDMETHODIMP CDecoder::SetOutBufSize(UInt32, UInt32 size)
{
_buffOutSize = size;
return S_OK;
}
HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress)
{
RINOK(CreateDecompressor());
ZSTDMT_RdWr_t rdwr;
size_t result;
UInt32 const toRead = static_cast < const UInt32 > (_buffInSize);
for(;;) {
UInt32 read;
HRESULT res = S_OK;
/* read input */
RINOK(inStream->Read(_buffIn, toRead, &read));
size_t InSize = static_cast < size_t > (read);
_processedIn += InSize;
struct ZstdStream Rd;
Rd.inStream = inStream;
Rd.processedIn = &_processedIn;
if (InSize == 0)
return S_OK;
struct ZstdStream Wr;
Wr.progress = progress;
Wr.outStream = outStream;
Wr.processedIn = &_processedIn;
Wr.processedOut = &_processedOut;
/* decompress input */
ZSTD_inBuffer input = { _buffIn, InSize, 0 };
for (;;) {
ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
result = ZSTD_decompressStream(_dstream, &output , &input);
#if 0
printf("%s in=%d out=%d result=%d in.pos=%d in.size=%d\n", __FUNCTION__,
InSize, output.pos, result, input.pos, input.size);
fflush(stdout);
#endif
if (ZSTD_isError(result))
/* 1) setup read/write functions */
rdwr.fn_read = ::ZstdRead;
rdwr.fn_write = ::ZstdWrite;
rdwr.arg_read = (void *)&Rd;
rdwr.arg_write = (void *)&Wr;
/* 2) create decompression context */
ZSTDMT_DCtx *ctx = ZSTDMT_createDCtx(_numThreads, _inputSize);
if (!ctx)
return S_FALSE;
/* 3) decompress */
result = ZSTDMT_decompressDCtx(ctx, &rdwr);
//printf("decompress = %d / %d\n", result, ZSTDMT_error_read_fail);
if (result == (size_t)-ZSTDMT_error_read_fail)
res = E_ABORT;
else if (ZSTDMT_isError(result))
return ErrorOut(result);
/* write decompressed stream and update progress */
RINOK(WriteStream(outStream, _buffOut, output.pos));
_processedOut += output.pos;
RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
/* one more round */
if ((input.pos == input.size) && (result == 1) && output.pos)
continue;
/* finished */
if (input.pos == input.size)
break;
/* end of frame */
if (result == 0) {
result = ZSTD_initDStream(_dstream);
if (ZSTD_isError(result))
return ErrorOut(result);
}
}
}
/* 4) free resources */
ZSTDMT_freeDCtx(ctx);
return res;
}
STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
@@ -221,69 +165,14 @@ STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
STDMETHODIMP CDecoder::ReleaseInStream()
{
_inStream.Release();
return S_OK;
}
STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize)
{
if (processedSize)
*processedSize = 0;
size_t result;
if (!_dstream)
if (CreateDecompressor() != S_OK)
return E_FAIL;
UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
Byte *dataout = static_cast < Byte* > (data);
for(;;) {
/* read input */
RINOK(_inStream->Read(_buffIn, toRead, &read));
size_t InSize = static_cast < size_t > (read);
_processedIn += InSize;
if (InSize == 0) {
return S_OK;
}
/* decompress input */
ZSTD_inBuffer input = { _buffIn, InSize, 0 };
for (;;) {
ZSTD_outBuffer output = { dataout, _buffOutSize, 0 };
result = ZSTD_decompressStream(_dstream, &output , &input);
if (ZSTD_isError(result))
return ErrorOut(result);
if (processedSize)
*processedSize += static_cast < UInt32 > (output.pos);
dataout += output.pos;
/* one more round */
if ((input.pos == input.size) && (result == 1) && output.pos)
continue;
/* finished */
if (input.pos == input.size)
break;
/* end of frame */
if (result == 0) {
result = ZSTD_initDStream(_dstream);
if (ZSTD_isError(result))
return ErrorOut(result);
}
}
}
}
#endif
HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
{
RINOK(SetOutStreamSizeResume(outSize));
return CodeSpec(_inStream, outStream, progress);
}
#endif
}}

54
CPP/7zip/Compress/ZstdDecoder.h
View File

@@ -1,22 +1,36 @@
// ZstdDecoder.h
// (C) 2016 Tino Reichardt
/**
* you can define ZSTD_LEGACY_SUPPORT to be backwards compatible
* with these versions: 0.5, 0.6, 0.7, 0.8 (0.8 == 1.0)
*
* /TR 2016-09-04
* you can define ZSTD_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7)
* /TR 2016-10-01
*/
#define ZSTD_STATIC_LINKING_ONLY
#include "../../../C/Alloc.h"
#include "../../../C/ZStd/zstd.h"
#include "../../../C/Threads.h"
#include "../../../C/zstd/zstd.h"
#include "../../../C/zstdmt/zstdmt.h"
#include "../../Windows/System.h"
#include "../../Common/Common.h"
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../Common/StreamUtils.h"
#include "../Common/RegisterCodec.h"
#include "../Common/ProgressMt.h"
struct ZstdStream {
ISequentialInStream *inStream;
ISequentialOutStream *outStream;
ICompressProgressInfo *progress;
UInt64 *processedIn;
UInt64 *processedOut;
CCriticalSection *cs;
int flags;
};
extern int ZstdRead(void *Stream, ZSTDMT_Buffer * in);
extern int ZstdWrite(void *Stream, ZSTDMT_Buffer * in);
namespace NCompress {
namespace NZSTD {
@@ -39,30 +53,19 @@ struct DProps
};
class CDecoder:public ICompressCoder,
public ICompressSetDecoderProperties2, public ICompressSetBufSize,
#ifndef NO_READ_FROM_CODER
public ICompressSetInStream,
public ICompressSetOutStreamSize, public ISequentialInStream,
#endif
public ICompressSetDecoderProperties2,
public CMyUnknownImp
{
CMyComPtr < ISequentialInStream > _inStream;
DProps _props;
ZSTD_DStream *_dstream;
void *_buffIn;
void *_buffOut;
size_t _buffInSize;
size_t _buffOutSize;
size_t _buffInSizeAllocated;
size_t _buffOutSizeAllocated;
CCriticalSection cs;
UInt64 _processedIn;
UInt64 _processedOut;
UInt32 _inputSize;
UInt32 _numThreads;
HRESULT CDecoder::CreateDecompressor();
HRESULT CDecoder::ErrorOut(size_t code);
HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
@@ -71,11 +74,8 @@ public:
MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2)
MY_QUERYINTERFACE_ENTRY (ICompressSetBufSize)
#ifndef NO_READ_FROM_CODER
MY_QUERYINTERFACE_ENTRY(ICompressSetInStream)
MY_QUERYINTERFACE_ENTRY (ICompressSetOutStreamSize)
MY_QUERYINTERFACE_ENTRY (ISequentialInStream)
#endif
MY_QUERYINTERFACE_END
@@ -83,16 +83,14 @@ public:
STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size);
STDMETHOD (SetOutStreamSize)(const UInt64 *outSize);
STDMETHOD (SetInBufSize) (UInt32 streamIndex, UInt32 size);
STDMETHOD (SetOutBufSize) (UInt32 streamIndex, UInt32 size);
STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads);
#ifndef NO_READ_FROM_CODER
STDMETHOD (SetInStream)(ISequentialInStream *inStream);
STDMETHOD (ReleaseInStream)();
STDMETHOD (Read) (void *data, UInt32 size, UInt32 *processedSize);
HRESULT CodeResume (ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
UInt64 GetInputProcessedSize() const { return _processedIn; }
#endif
HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
CDecoder();
virtual ~CDecoder();

127
CPP/7zip/Compress/ZstdEncoder.cpp
View File

@@ -1,32 +1,27 @@
// ZstdEncoder.cpp
// (C) 2016 Tino Reichardt
#include "StdAfx.h"
#include "ZstdEncoder.h"
#include "ZstdDecoder.h"
#ifndef EXTRACT_ONLY
namespace NCompress {
namespace NZSTD {
CEncoder::CEncoder():
_cstream(NULL),
_buffIn(NULL),
_buffOut(NULL),
_buffInSize(0),
_buffOutSize(0),
_processedIn(0),
_processedOut(0)
_processedOut(0),
_inputSize(0),
_ctx(NULL),
_numThreads(NWindows::NSystem::GetNumberOfProcessors())
{
_props.clear();
}
CEncoder::~CEncoder()
{
if (_cstream)
ZSTD_freeCStream(_cstream);
MyFree(_buffIn);
MyFree(_buffOut);
if (_ctx)
ZSTDMT_freeCCtx(_ctx);
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
@@ -37,6 +32,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
{
const PROPVARIANT & prop = coderProps[i];
PROPID propID = propIDs[i];
UInt32 v = (UInt32)prop.ulVal;
switch (propID)
{
case NCoderPropID::kLevel:
@@ -52,6 +48,11 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
break;
}
case NCoderPropID::kNumThreads:
{
SetNumberOfThreads(v);
break;
}
default:
{
break;
@@ -59,9 +60,6 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
}
}
_processedIn = 0;
_processedOut = 0;
return S_OK;
}
@@ -74,72 +72,67 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 * /*inSize*/ ,
const UInt64 * /*outSize */, ICompressProgressInfo *progress)
{
ZSTDMT_RdWr_t rdwr;
size_t result;
HRESULT res = S_OK;
/* init only once in beginning */
if (!_cstream) {
struct ZstdStream Rd;
Rd.inStream = inStream;
Rd.outStream = outStream;
Rd.processedIn = &_processedIn;
Rd.processedOut = &_processedOut;
/* allocate stream */
_cstream = ZSTD_createCStream();
if (!_cstream)
return E_OUTOFMEMORY;
struct ZstdStream Wr;
if (_processedIn == 0)
Wr.progress = progress;
else
Wr.progress = 0;
Wr.inStream = inStream;
Wr.outStream = outStream;
Wr.processedIn = &_processedIn;
Wr.processedOut = &_processedOut;
/* allocate buffers */
_buffInSize = ZSTD_CStreamInSize();
_buffIn = MyAlloc(_buffInSize);
if (!_buffIn)
return E_OUTOFMEMORY;
/* 1) setup read/write functions */
rdwr.fn_read = ::ZstdRead;
rdwr.fn_write = ::ZstdWrite;
rdwr.arg_read = (void *)&Rd;
rdwr.arg_write = (void *)&Wr;
_buffOutSize = ZSTD_CStreamOutSize();
_buffOut = MyAlloc(_buffOutSize);
if (!_buffOut)
return E_OUTOFMEMORY;
}
/* init or re-init stream */
result = ZSTD_initCStream(_cstream, _props._level);
if (ZSTD_isError(result))
/* 2) create compression context, if needed */
if (!_ctx)
_ctx = ZSTDMT_createCCtx(_numThreads, _props._level, _inputSize);
if (!_ctx)
return S_FALSE;
UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
for(;;) {
/* 3) compress */
result = ZSTDMT_compressCCtx(_ctx, &rdwr);
if (result == (size_t)-ZSTDMT_error_read_fail)
res = E_ABORT;
else if (ZSTDMT_isError(result))
return ErrorOut(result);
/* read input */
RINOK(inStream->Read(_buffIn, toRead, &read));
size_t InSize = static_cast < size_t > (read);
_processedIn += InSize;
if (InSize == 0) {
/* @eof */
ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
result = ZSTD_endStream(_cstream, &output);
if (ZSTD_isError(result))
return S_FALSE;
if (output.pos) {
/* write last compressed bytes and update progress */
RINOK(WriteStream(outStream, _buffOut, output.pos));
_processedOut += output.pos;
RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
return res;
}
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
{
const UInt32 kNumThreadsMax = ZSTDMT_THREAD_MAX;
if (numThreads < 1) numThreads = 1;
if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
_numThreads = numThreads;
return S_OK;
}
/* compress input */
ZSTD_inBuffer input = { _buffIn, InSize, 0 };
while (input.pos < input.size) {
ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
result = ZSTD_compressStream(_cstream, &output , &input);
if (ZSTD_isError(result))
HRESULT CEncoder::ErrorOut(size_t code)
{
const char *strError = ZSTDMT_getErrorString(code);
wchar_t wstrError[200+5]; /* no malloc here, /TR */
mbstowcs(wstrError, strError, 200);
MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
MyFree(wstrError);
return S_FALSE;
/* write compressed stream and update progress */
RINOK(WriteStream(outStream, _buffOut, output.pos));
_processedOut += output.pos;
RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
}
}
}
}}

16
CPP/7zip/Compress/ZstdEncoder.h
View File

@@ -1,9 +1,10 @@
// ZstdEncoder.h
// (C) 2016 Tino Reichardt
#define ZSTD_STATIC_LINKING_ONLY
#include "../../../C/Alloc.h"
#include "../../../C/ZStd/zstd.h"
#include "../../../C/Threads.h"
#include "../../../C/zstd/zstd.h"
#include "../../../C/zstdmt/zstdmt.h"
#include "../../Common/Common.h"
#include "../../Common/MyCom.h"
@@ -39,21 +40,20 @@ class CEncoder:
{
CProps _props;
ZSTD_CStream *_cstream;
void *_buffIn;
void *_buffOut;
size_t _buffInSize;
size_t _buffOutSize;
UInt64 _processedIn;
UInt64 _processedOut;
UInt32 _inputSize;
UInt32 _numThreads;
HRESULT CreateCompressor();
ZSTDMT_CCtx *_ctx;
HRESULT CEncoder::ErrorOut(size_t code);
public:
MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream);
STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads);
CEncoder();
virtual ~CEncoder();

1
CPP/7zip/Compress/ZstdRegister.cpp
View File

@@ -1,4 +1,3 @@
// ZstdRegister.cpp
// (C) 2016 Tino Reichardt
#include "StdAfx.h"

5
CPP/7zip/Guid.txt
View File

@@ -161,8 +161,11 @@ Handler GUIDs:
0A lzma
0B lzma86
0C xz
0D ppmd
0E zstd
0F lz4
10 lz5
C6 Lzip
C7 Ext
C8 VMDK
C9 VDI

1
CPP/7zip/UI/Common/OpenArchive.cpp
View File

@@ -1066,6 +1066,7 @@ static const char * const k_Formats_with_simple_signuature[] =
, "rar"
, "bzip2"
, "gzip"
, "lzip"
, "cab"
, "wim"
, "rpm"

4
CPP/7zip/UI/Explorer/ContextMenu.cpp
View File

@@ -283,8 +283,12 @@ static const char * const kArcExts[] =
"7z"
, "bz2"
, "gz"
, "lz"
, "lz4"
, "lz5"
, "rar"
, "zip"
, "zst"
};
static bool IsItArcExt(const UString &ext)

204
CPP/7zip/UI/GUI/CompressDialog.cpp
View File

@@ -87,13 +87,9 @@ static const UInt32 g_Levels[] =
{
IDS_METHOD_STORE,
IDS_METHOD_FASTEST,
0,
IDS_METHOD_FAST,
0,
IDS_METHOD_NORMAL,
0,
IDS_METHOD_MAXIMUM,
0,
IDS_METHOD_ULTRA
};
@@ -101,6 +97,8 @@ enum EMethodID
{
kCopy,
kZSTD,
kLZ4,
kLZ5,
kLZMA,
kLZMA2,
kPPMd,
@@ -114,6 +112,8 @@ static const LPCWSTR kMethodsNames[] =
{
L"Copy",
L"ZSTD",
L"LZ4",
L"LZ5",
L"LZMA",
L"LZMA2",
L"PPMd",
@@ -126,6 +126,8 @@ static const LPCWSTR kMethodsNames[] =
static const EMethodID g_7zMethods[] =
{
kZSTD,
kLZ4,
kLZ5,
kLZMA2,
kLZMA,
kPPMd,
@@ -167,6 +169,21 @@ static const EMethodID g_XzMethods[] =
kLZMA2
};
static const EMethodID g_ZstdMethods[] =
{
kZSTD
};
static const EMethodID g_Lz4Methods[] =
{
kLZ4
};
static const EMethodID g_Lz5Methods[] =
{
kLZ5
};
static const EMethodID g_SwfcMethods[] =
{
kDeflate
@@ -227,6 +244,24 @@ static const CFormatInfo g_Formats[] =
METHODS_PAIR(g_XzMethods),
false, false, true, false, false, false
},
{
L"zstd",
(1 << 0) | (1 << 1) | (1 << 5) | (1 << 11) | (1 << 17) | (1 << 22),
METHODS_PAIR(g_ZstdMethods),
false, false, true, false, false, false
},
{
L"lz4",
(1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 16),
METHODS_PAIR(g_Lz4Methods),
false, false, true, false, false, false
},
{
L"lz5",
(1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 16),
METHODS_PAIR(g_Lz5Methods),
false, false, true, false, false, false
},
{
L"Swfc",
(1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9),
@@ -867,10 +902,6 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam)
}
case IDC_COMPRESS_LEVEL:
{
const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
int index = FindRegistryFormatAlways(ai.Name);
NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
fo.ResetForLevelChange();
SetMethod(GetMethodID());
SetSolidBlockSize();
SetNumThreads();
@@ -880,13 +911,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam)
}
case IDC_COMPRESS_METHOD:
{
// MessageBoxW(*this, L"IDC_COMPRESS_METHOD!", L"7-Zip", MB_ICONERROR);
if (GetMethodID() == kZSTD)
{
SetLevel_zstd();
} else {
SetLevel_default();
}
SetLevel();
SetDictionary();
SetOrder();
SetSolidBlockSize();
@@ -1018,10 +1043,13 @@ void CCompressDialog::SetNearestSelectComboBox(NControl::CComboBox &comboBox, UI
comboBox.SetCurSel(0);
}
void CCompressDialog::SetLevel_zstd()
void CCompressDialog::SetLevel()
{
UInt32 level = GetLevel2();
UInt32 LevelsMask;
UInt32 langID = 0;
SetMethod(GetMethodID());
const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
{
int index = FindRegistryFormat(ai.Name);
@@ -1030,111 +1058,52 @@ void CCompressDialog::SetLevel_zstd()
const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
if (fo.Level <= 22)
level = fo.Level;
else
level = 5;
}
}
/* ZStandard has 22 levels */
m_Level.ResetContent();
if (GetMethodID() == kZSTD)
LevelsMask = g_Formats[6].LevelsMask;
else if (GetMethodID() == kLZ4)
LevelsMask = g_Formats[7].LevelsMask;
else if (GetMethodID() == kLZ5)
LevelsMask = g_Formats[8].LevelsMask;
else
LevelsMask = g_Formats[GetStaticFormatIndex()].LevelsMask;
for (unsigned i = 0; i <= 22; i++)
{
TCHAR s[40];
TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 };
ConvertUInt32ToString(i, s);
lstrcat(t, s);
switch (i) {
case 0:
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(IDS_METHOD_STORE));
lstrcat(t, TEXT(")"));
break;
case 1:
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(IDS_METHOD_FASTEST));
lstrcat(t, TEXT(")"));
break;
case 5:
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(IDS_METHOD_FAST));
lstrcat(t, TEXT(")"));
break;
case 11:
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(IDS_METHOD_NORMAL));
lstrcat(t, TEXT(")"));
break;
case 17:
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(IDS_METHOD_MAXIMUM));
lstrcat(t, TEXT(")"));
break;
case 22:
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(IDS_METHOD_ULTRA));
lstrcat(t, TEXT(")"));
break;
}
int index = (int)m_Level.AddString(t);
m_Level.SetItemData(index, i);
}
SetNearestSelectComboBox(m_Level, level);
return;
}
// max reached
if (LevelsMask < (UInt32)(1 << i))
break;
void CCompressDialog::SetLevel_default()
if ((LevelsMask & (1 << i)) != 0)
{
UInt32 level = GetLevel2();
const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
{
int index = FindRegistryFormat(ai.Name);
if (index >= 0)
{
const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
if (fo.Level <= 9)
level = fo.Level;
else
level = 9;
}
}
/* 9 default levels */
m_Level.ResetContent();
const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()];
for (unsigned i = 0; i <= 9; i++)
{
if ((fi.LevelsMask & (1 << i)) != 0)
{
TCHAR s[40];
TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 };
UInt32 langID = g_Levels[i];
ConvertUInt32ToString(i, s);
lstrcat(t, s);
lstrcat(t, TEXT(" ("));
lstrcat(t, LangString(langID));
lstrcat(t, LangString(g_Levels[langID]));
lstrcat(t, TEXT(")"));
int index = (int)m_Level.AddString(t);
m_Level.SetItemData(index, i);
}
}
SetNearestSelectComboBox(m_Level, level);
return;
}
void CCompressDialog::SetLevel()
{
SetMethod();
if (GetMethodID() == kZSTD)
{
SetLevel_zstd();
langID++;
} else {
SetLevel_default();
ConvertUInt32ToString(i, s);
lstrcat(t, s);
int index = (int)m_Level.AddString(t);
m_Level.SetItemData(index, i);
}
}
SetNearestSelectComboBox(m_Level, level);
return;
}
void CCompressDialog::SetMethod(int keepMethodId)
{
m_Method.ResetContent();
@@ -1645,6 +1614,9 @@ void CCompressDialog::SetNumThreads()
int methodID = GetMethodID();
switch (methodID)
{
case kLZ4: numAlgoThreadsMax = 128; break;
case kLZ5: numAlgoThreadsMax = 128; break;
case kZSTD: numAlgoThreadsMax = 128; break;
case kLZMA: numAlgoThreadsMax = 2; break;
case kLZMA2: numAlgoThreadsMax = 32; break;
case kBZip2: numAlgoThreadsMax = 32; break;
@@ -1741,38 +1713,6 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory)
return size + decompressMemory;
}
case kZSTD:
{
/* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=131075 */
size = 0;
switch (level) {
case 1: size = 824228; decompressMemory = 415024; return size;
case 2: size = 1282980; decompressMemory = 415024; return size;
case 3: size = 922532; decompressMemory = 415024; return size;
case 4: size = 1414052; decompressMemory = 415024; return size;
case 5: size = 1545124; decompressMemory = 415024; return size;
case 6: size = 1807268; decompressMemory = 415024; return size;
case 7: size = 1807268; decompressMemory = 415024; return size;
case 8: size = 1807268; decompressMemory = 415024; return size;
case 9: size = 1807268; decompressMemory = 415024; return size;
case 10: size = 1807268; decompressMemory = 415024; return size;
case 11: size = 2331556; decompressMemory = 415024; return size;
case 12: size = 2331556; decompressMemory = 415024; return size;
case 13: size = 3380132; decompressMemory = 415024; return size;
case 14: size = 3004832; decompressMemory = 415024; return size;
case 15: size = 3004832; decompressMemory = 415024; return size;
case 16: size = 4697834; decompressMemory = 415024; return size;
case 17: size = 4697834; decompressMemory = 415024; return size;
case 18: size = 4697834; decompressMemory = 415024; return size;
case 19: size = 4697834; decompressMemory = 415024; return size;
case 20: size = 4697834; decompressMemory = 415024; return size;
case 21: size = 4697834; decompressMemory = 415024; return size;
case 22: size = 4697834; decompressMemory = 415024; return size;
}
decompressMemory = 0;
return size;
}
case kDeflate:
case kDeflate64:
{

2
CPP/7zip/UI/GUI/CompressDialog.h
View File

@@ -130,8 +130,6 @@ class CCompressDialog: public NWindows::NControl::CModalDialog
void SetNearestSelectComboBox(NWindows::NControl::CComboBox &comboBox, UInt32 value);
void SetLevel();
void SetLevel_zstd();
void SetLevel_default();
void SetMethod(int keepMethodId = -1);
int GetMethodID();

Some files were not shown because too many files have changed in this diff Show More