merge multi threading to master branch

- updated zstd to latest devel release
- lz4, lz5 and zstd is included now
- all three support threading

C/7zVersion.h

@@ -3,12 +3,12 @@
 #define MY_VER_BUILD 0
 #define MY_VERSION_NUMBERS "16.04 ZS"
 #define MY_VERSION "16.04 ZS"
-#define MY_DATE "2016-10-06"
+#define MY_DATE "2016-10-16"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"
 #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
-#define MY_COPYRIGHT_CR "(c) 1999-2016 Igor Pavlov, Tino Reichardt"
+#define MY_COPYRIGHT_CR "Copyright (c) 1999-2016 Igor Pavlov"
 
 #ifdef USE_COPYRIGHT_CR
   #define MY_COPYRIGHT MY_COPYRIGHT_CR

C/7zVersionTr.h

@@ -3,7 +3,7 @@
 #define MY_VER_BUILD 0
 #define MY_VERSION_NUMBERS "1.1.0"
 #define MY_VERSION "1.1.0"
-#define MY_DATE "2016-10-06"
+#define MY_DATE "2016-10-16"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME "Tino Reichardt"

C/lz4/lz4.h

@@ -0,0 +1,360 @@
+/*
+   LZ4 - Fast LZ compression algorithm
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * lz4.h provides block compression functions, and gives full buffer control to programmer.
+ * If you need to generate inter-operable compressed data (respecting LZ4 frame specification),
+ * and can let the library handle its own memory, please use lz4frame.h instead.
+*/
+
+/**************************************
+*  Version
+**************************************/
+#define LZ4_VERSION_MAJOR    1    /* for breaking interface changes  */
+#define LZ4_VERSION_MINOR    7    /* for new (non-breaking) interface capabilities */
+#define LZ4_VERSION_RELEASE  1    /* for tweaks, bug-fixes, or development */
+#define LZ4_VERSION_NUMBER (LZ4_VERSION_MAJOR *100*100 + LZ4_VERSION_MINOR *100 + LZ4_VERSION_RELEASE)
+int LZ4_versionNumber (void);
+
+/**************************************
+*  Tuning parameter
+**************************************/
+/*
+ * LZ4_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ * Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache
+ */
+#define LZ4_MEMORY_USAGE 14
+
+
+/**************************************
+*  Simple Functions
+**************************************/
+
+int LZ4_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
+int LZ4_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+
+/*
+LZ4_compress_default() :
+    Compresses 'sourceSize' bytes from buffer 'source'
+    into already allocated 'dest' buffer of size 'maxDestSize'.
+    Compression is guaranteed to succeed if 'maxDestSize' >= LZ4_compressBound(sourceSize).
+    It also runs faster, so it's a recommended setting.
+    If the function cannot compress 'source' into a more limited 'dest' budget,
+    compression stops *immediately*, and the function result is zero.
+    As a consequence, 'dest' content is not valid.
+    This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
+        sourceSize  : Max supported value is LZ4_MAX_INPUT_VALUE
+        maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
+        return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
+              or 0 if compression fails
+
+LZ4_decompress_safe() :
+    compressedSize : is the precise full size of the compressed block.
+    maxDecompressedSize : is the size of destination buffer, which must be already allocated.
+    return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
+             If destination buffer is not large enough, decoding will stop and output an error code (<0).
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             This function is protected against buffer overflow exploits, including malicious data packets.
+             It never writes outside output buffer, nor reads outside input buffer.
+*/
+
+
+/**************************************
+*  Advanced Functions
+**************************************/
+#define LZ4_MAX_INPUT_SIZE        0x7E000000   /* 2 113 929 216 bytes */
+#define LZ4_COMPRESSBOUND(isize)  ((unsigned)(isize) > (unsigned)LZ4_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/255) + 16)
+
+/*
+LZ4_compressBound() :
+    Provides the maximum size that LZ4 compression may output in a "worst case" scenario (input data not compressible)
+    This function is primarily useful for memory allocation purposes (destination buffer size).
+    Macro LZ4_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
+    Note that LZ4_compress_default() compress faster when dest buffer size is >= LZ4_compressBound(srcSize)
+        inputSize  : max supported value is LZ4_MAX_INPUT_SIZE
+        return : maximum output size in a "worst case" scenario
+              or 0, if input size is too large ( > LZ4_MAX_INPUT_SIZE)
+*/
+int LZ4_compressBound(int inputSize);
+
+/*
+LZ4_compress_fast() :
+    Same as LZ4_compress_default(), but allows to select an "acceleration" factor.
+    The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
+    It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
+    An acceleration value of "1" is the same as regular LZ4_compress_default()
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz4.c), which is 1.
+*/
+int LZ4_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+
+
+/*
+LZ4_compress_fast_extState() :
+    Same compression function, just using an externally allocated memory space to store compression state.
+    Use LZ4_sizeofState() to know how much memory must be allocated,
+    and allocate it on 8-bytes boundaries (using malloc() typically).
+    Then, provide it as 'void* state' to compression function.
+*/
+int LZ4_sizeofState(void);
+int LZ4_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
+
+
+/*
+LZ4_compress_destSize() :
+    Reverse the logic, by compressing as much data as possible from 'source' buffer
+    into already allocated buffer 'dest' of size 'targetDestSize'.
+    This function either compresses the entire 'source' content into 'dest' if it's large enough,
+    or fill 'dest' buffer completely with as much data as possible from 'source'.
+        *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
+                         New value is necessarily <= old value.
+        return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
+              or 0 if compression fails
+*/
+int LZ4_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
+
+
+/*
+LZ4_decompress_fast() :
+    originalSize : is the original and therefore uncompressed size
+    return : the number of bytes read from the source buffer (in other words, the compressed size)
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
+    note : This function fully respect memory boundaries for properly formed compressed data.
+           It is a bit faster than LZ4_decompress_safe().
+           However, it does not provide any protection against intentionally modified data stream (malicious input).
+           Use this function in trusted environment only (data to decode comes from a trusted source).
+*/
+int LZ4_decompress_fast (const char* source, char* dest, int originalSize);
+
+/*
+LZ4_decompress_safe_partial() :
+    This function decompress a compressed block of size 'compressedSize' at position 'source'
+    into destination buffer 'dest' of size 'maxDecompressedSize'.
+    The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+    reducing decompression time.
+    return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
+       Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+             Always control how many bytes were decoded.
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+int LZ4_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
+
+
+/***********************************************
+*  Streaming Compression Functions
+***********************************************/
+#define LZ4_STREAMSIZE_U64 ((1 << (LZ4_MEMORY_USAGE-3)) + 4)
+#define LZ4_STREAMSIZE     (LZ4_STREAMSIZE_U64 * sizeof(long long))
+/*
+ * LZ4_stream_t
+ * information structure to track an LZ4 stream.
+ * important : init this structure content before first use !
+ * note : only allocated directly the structure if you are statically linking LZ4
+ *        If you are using liblz4 as a DLL, please use below construction methods instead.
+ */
+typedef struct { long long table[LZ4_STREAMSIZE_U64]; } LZ4_stream_t;
+
+/*
+ * LZ4_resetStream
+ * Use this function to init an allocated LZ4_stream_t structure
+ */
+void LZ4_resetStream (LZ4_stream_t* streamPtr);
+
+/*
+ * LZ4_createStream will allocate and initialize an LZ4_stream_t structure
+ * LZ4_freeStream releases its memory.
+ * In the context of a DLL (liblz4), please use these methods rather than the static struct.
+ * They are more future proof, in case of a change of LZ4_stream_t size.
+ */
+LZ4_stream_t* LZ4_createStream(void);
+int           LZ4_freeStream (LZ4_stream_t* streamPtr);
+
+/*
+ * LZ4_loadDict
+ * Use this function to load a static dictionary into LZ4_stream.
+ * Any previous data will be forgotten, only 'dictionary' will remain in memory.
+ * Loading a size of 0 is allowed.
+ * Return : dictionary size, in bytes (necessarily <= 64 KB)
+ */
+int LZ4_loadDict (LZ4_stream_t* streamPtr, const char* dictionary, int dictSize);
+
+/*
+ * LZ4_compress_fast_continue
+ * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
+ * Important : Previous data blocks are assumed to still be present and unmodified !
+ * 'dst' buffer must be already allocated.
+ * If maxDstSize >= LZ4_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ */
+int LZ4_compress_fast_continue (LZ4_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+
+/*
+ * LZ4_saveDict
+ * If previously compressed data block is not guaranteed to remain available at its memory location
+ * save it into a safer place (char* safeBuffer)
+ * Note : you don't need to call LZ4_loadDict() afterwards,
+ *        dictionary is immediately usable, you can therefore call LZ4_compress_fast_continue()
+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
+ */
+int LZ4_saveDict (LZ4_stream_t* streamPtr, char* safeBuffer, int dictSize);
+
+
+/************************************************
+*  Streaming Decompression Functions
+************************************************/
+
+#define LZ4_STREAMDECODESIZE_U64  4
+#define LZ4_STREAMDECODESIZE     (LZ4_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+typedef struct { unsigned long long table[LZ4_STREAMDECODESIZE_U64]; } LZ4_streamDecode_t;
+/*
+ * LZ4_streamDecode_t
+ * information structure to track an LZ4 stream.
+ * init this structure content using LZ4_setStreamDecode or memset() before first use !
+ *
+ * In the context of a DLL (liblz4) please prefer usage of construction methods below.
+ * They are more future proof, in case of a change of LZ4_streamDecode_t size in the future.
+ * LZ4_createStreamDecode will allocate and initialize an LZ4_streamDecode_t structure
+ * LZ4_freeStreamDecode releases its memory.
+ */
+LZ4_streamDecode_t* LZ4_createStreamDecode(void);
+int                 LZ4_freeStreamDecode (LZ4_streamDecode_t* LZ4_stream);
+
+/*
+ * LZ4_setStreamDecode
+ * Use this function to instruct where to find the dictionary.
+ * Setting a size of 0 is allowed (same effect as reset).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ4_setStreamDecode (LZ4_streamDecode_t* LZ4_streamDecode, const char* dictionary, int dictSize);
+
+/*
+*_continue() :
+    These decoding functions allow decompression of multiple blocks in "streaming" mode.
+    Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+    In the case of a ring buffers, decoding buffer must be either :
+    - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+      In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+    - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+      maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+      In which case, encoding and decoding buffers do not need to be synchronized,
+      and encoding ring buffer can have any size, including small ones ( < 64 KB).
+    - _At least_ 64 KB + 8 bytes + maxBlockSize.
+      In which case, encoding and decoding buffers do not need to be synchronized,
+      and encoding ring buffer can have any size, including larger than decoding buffer.
+    Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+    and indicate where it is saved using LZ4_setStreamDecode()
+*/
+int LZ4_decompress_safe_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+int LZ4_decompress_fast_continue (LZ4_streamDecode_t* LZ4_streamDecode, const char* source, char* dest, int originalSize);
+
+
+/*
+Advanced decoding functions :
+*_usingDict() :
+    These decoding functions work the same as
+    a combination of LZ4_setStreamDecode() followed by LZ4_decompress_x_continue()
+    They are stand-alone. They don't need nor update an LZ4_streamDecode_t structure.
+*/
+int LZ4_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
+int LZ4_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+
+
+
+/**************************************
+*  Obsolete Functions
+**************************************/
+/* Deprecate Warnings */
+/* Should these warnings messages be a problem,
+   it is generally possible to disable them,
+   with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual for example.
+   You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
+#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ4_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#  if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (LZ4_GCC_VERSION >= 301)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define LZ4_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
+#    define LZ4_DEPRECATED(message)
+#  endif
+#endif /* LZ4_DEPRECATE_WARNING_DEFBLOCK */
+
+/* Obsolete compression functions */
+/* These functions are planned to start generate warnings by r131 approximately */
+int LZ4_compress               (const char* source, char* dest, int sourceSize);
+int LZ4_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+int LZ4_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compress_continue                (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize);
+int LZ4_compress_limitedOutput_continue  (LZ4_stream_t* LZ4_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Obsolete decompression functions */
+/* These function names are completely deprecated and must no longer be used.
+   They are only provided here for compatibility with older programs.
+    - LZ4_uncompress is the same as LZ4_decompress_fast
+    - LZ4_uncompress_unknownOutputSize is the same as LZ4_decompress_safe
+   These function prototypes are now disabled; uncomment them only if you really need them.
+   It is highly recommended to stop using these prototypes and migrate to maintained ones */
+/* int LZ4_uncompress (const char* source, char* dest, int outputSize); */
+/* int LZ4_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ4_DEPRECATED("use LZ4_createStream() instead") void* LZ4_create (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_createStream() instead") int   LZ4_sizeofStreamState(void);
+LZ4_DEPRECATED("use LZ4_resetStream() instead")  int   LZ4_resetStreamState(void* state, char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDict() instead")     char* LZ4_slideInputBuffer (void* state);
+
+/* Obsolete streaming decoding functions */
+LZ4_DEPRECATED("use LZ4_decompress_safe_usingDict() instead") int LZ4_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ4_DEPRECATED("use LZ4_decompress_fast_usingDict() instead") int LZ4_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz4/lz4frame.h

@@ -0,0 +1,303 @@
+/*
+   LZ4 auto-framing library
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+/* LZ4F is a stand-alone API to create LZ4-compressed frames
+ * fully conformant to specification v1.5.1.
+ * All related operations, including memory management, are handled by the library.
+ * You don't need lz4.h when using lz4frame.h.
+ * */
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/**************************************
+*  Includes
+**************************************/
+#include <stddef.h>   /* size_t */
+
+
+/**************************************
+ * Error management
+ * ************************************/
+typedef size_t LZ4F_errorCode_t;
+
+unsigned    LZ4F_isError(LZ4F_errorCode_t code);
+const char* LZ4F_getErrorName(LZ4F_errorCode_t code);   /* return error code string; useful for debugging */
+
+
+/**************************************
+ * Frame compression types
+ * ************************************/
+//#define LZ4F_DISABLE_OBSOLETE_ENUMS
+#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+#  define LZ4F_OBSOLETE_ENUM(x) ,x
+#else
+#  define LZ4F_OBSOLETE_ENUM(x)
+#endif
+
+typedef enum {
+    LZ4F_default=0,
+    LZ4F_max64KB=4,
+    LZ4F_max256KB=5,
+    LZ4F_max1MB=6,
+    LZ4F_max4MB=7
+    LZ4F_OBSOLETE_ENUM(max64KB = LZ4F_max64KB)
+    LZ4F_OBSOLETE_ENUM(max256KB = LZ4F_max256KB)
+    LZ4F_OBSOLETE_ENUM(max1MB = LZ4F_max1MB)
+    LZ4F_OBSOLETE_ENUM(max4MB = LZ4F_max4MB)
+} LZ4F_blockSizeID_t;
+
+typedef enum {
+    LZ4F_blockLinked=0,
+    LZ4F_blockIndependent
+    LZ4F_OBSOLETE_ENUM(blockLinked = LZ4F_blockLinked)
+    LZ4F_OBSOLETE_ENUM(blockIndependent = LZ4F_blockIndependent)
+} LZ4F_blockMode_t;
+
+typedef enum {
+    LZ4F_noContentChecksum=0,
+    LZ4F_contentChecksumEnabled
+    LZ4F_OBSOLETE_ENUM(noContentChecksum = LZ4F_noContentChecksum)
+    LZ4F_OBSOLETE_ENUM(contentChecksumEnabled = LZ4F_contentChecksumEnabled)
+} LZ4F_contentChecksum_t;
+
+typedef enum {
+    LZ4F_frame=0,
+    LZ4F_skippableFrame
+    LZ4F_OBSOLETE_ENUM(skippableFrame = LZ4F_skippableFrame)
+} LZ4F_frameType_t;
+
+#ifndef LZ4F_DISABLE_OBSOLETE_ENUMS
+typedef LZ4F_blockSizeID_t blockSizeID_t;
+typedef LZ4F_blockMode_t blockMode_t;
+typedef LZ4F_frameType_t frameType_t;
+typedef LZ4F_contentChecksum_t contentChecksum_t;
+#endif
+
+typedef struct {
+  LZ4F_blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+  LZ4F_blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
+  LZ4F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
+  LZ4F_frameType_t       frameType;             /* LZ4F_frame, skippableFrame ; 0 == default */
+  unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
+  unsigned               reserved[2];           /* must be zero for forward compatibility */
+} LZ4F_frameInfo_t;
+
+typedef struct {
+  LZ4F_frameInfo_t frameInfo;
+  int      compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
+  unsigned autoFlush;              /* 1 == always flush (reduce need for tmp buffer) */
+  unsigned reserved[4];            /* must be zero for forward compatibility */
+} LZ4F_preferences_t;
+
+
+/***********************************
+ * Simple compression function
+ * *********************************/
+size_t LZ4F_compressFrameBound(size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
+
+size_t LZ4F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_preferences_t* preferencesPtr);
+/* LZ4F_compressFrame()
+ * Compress an entire srcBuffer into a valid LZ4 frame, as defined by specification v1.5.1
+ * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
+ * You can get the minimum value of dstMaxSize by using LZ4F_compressFrameBound()
+ * If this condition is not respected, LZ4F_compressFrame() will fail (result is an errorCode)
+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+ * The result of the function is the number of bytes written into dstBuffer.
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ */
+
+
+
+/**********************************
+*  Advanced compression functions
+**********************************/
+typedef struct LZ4F_cctx_s* LZ4F_compressionContext_t;   /* must be aligned on 8-bytes */
+
+typedef struct {
+  unsigned stableSrc;    /* 1 == src content will remain available on future calls to LZ4F_compress(); avoid saving src content within tmp buffer as future dictionary */
+  unsigned reserved[3];
+} LZ4F_compressOptions_t;
+
+/* Resource Management */
+
+#define LZ4F_VERSION 100
+LZ4F_errorCode_t LZ4F_createCompressionContext(LZ4F_compressionContext_t* cctxPtr, unsigned version);
+LZ4F_errorCode_t LZ4F_freeCompressionContext(LZ4F_compressionContext_t cctx);
+/* LZ4F_createCompressionContext() :
+ * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ * This is achieved using LZ4F_createCompressionContext(), which takes as argument a version and an LZ4F_preferences_t structure.
+ * The version provided MUST be LZ4F_VERSION. It is intended to track potential version differences between different binaries.
+ * The function will provide a pointer to a fully allocated LZ4F_compressionContext_t object.
+ * If the result LZ4F_errorCode_t is not zero, there was an error during context creation.
+ * Object can release its memory using LZ4F_freeCompressionContext();
+ */
+
+
+/* Compression */
+
+size_t LZ4F_compressBegin(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_preferences_t* prefsPtr);
+/* LZ4F_compressBegin() :
+ * will write the frame header into dstBuffer.
+ * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
+ * The LZ4F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
+ * The result of the function is the number of bytes written into dstBuffer for the header
+ * or an error code (can be tested using LZ4F_isError())
+ */
+
+size_t LZ4F_compressBound(size_t srcSize, const LZ4F_preferences_t* prefsPtr);
+/* LZ4F_compressBound() :
+ * Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
+ * Different preferences can produce different results.
+ * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
+ * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
+ */
+
+size_t LZ4F_compressUpdate(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ4F_compressOptions_t* cOptPtr);
+/* LZ4F_compressUpdate()
+ * LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
+ * You can get the minimum value of dstMaxSize by using LZ4F_compressBound().
+ * If this condition is not respected, LZ4F_compress() will fail (result is an errorCode).
+ * LZ4F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ */
+
+size_t LZ4F_flush(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr);
+/* LZ4F_flush()
+ * Should you need to generate compressed data immediately, without waiting for the current block to be filled,
+ * you can call LZ4_flush(), which will immediately compress any remaining data buffered within cctx.
+ * Note that dstMaxSize must be large enough to ensure the operation will be successful.
+ * LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * The result of the function is the number of bytes written into dstBuffer
+ * (it can be zero, this means there was no data left within cctx)
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ */
+
+size_t LZ4F_compressEnd(LZ4F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ4F_compressOptions_t* cOptPtr);
+/* LZ4F_compressEnd()
+ * When you want to properly finish the compressed frame, just call LZ4F_compressEnd().
+ * It will flush whatever data remained within compressionContext (like LZ4_flush())
+ * but also properly finalize the frame, with an endMark and a checksum.
+ * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ * The function outputs an error code if it fails (can be tested using LZ4F_isError())
+ * The LZ4F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * A successful call to LZ4F_compressEnd() makes cctx available again for next compression task.
+ */
+
+
+/***********************************
+*  Decompression functions
+***********************************/
+
+typedef struct LZ4F_dctx_s* LZ4F_decompressionContext_t;   /* must be aligned on 8-bytes */
+
+typedef struct {
+  unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
+  unsigned reserved[3];
+} LZ4F_decompressOptions_t;
+
+
+/* Resource management */
+
+LZ4F_errorCode_t LZ4F_createDecompressionContext(LZ4F_decompressionContext_t* dctxPtr, unsigned version);
+LZ4F_errorCode_t LZ4F_freeDecompressionContext(LZ4F_decompressionContext_t dctx);
+/* LZ4F_createDecompressionContext() :
+ * The first thing to do is to create an LZ4F_decompressionContext_t object, which will be used in all decompression operations.
+ * This is achieved using LZ4F_createDecompressionContext().
+ * The version provided MUST be LZ4F_VERSION. It is intended to track potential breaking differences between different versions.
+ * The function will provide a pointer to a fully allocated and initialized LZ4F_decompressionContext_t object.
+ * The result is an errorCode, which can be tested using LZ4F_isError().
+ * dctx memory can be released using LZ4F_freeDecompressionContext();
+ * The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ * That is, it should be == 0 if decompression has been completed fully and correctly.
+ */
+
+
+/* Decompression */
+
+size_t LZ4F_getFrameInfo(LZ4F_decompressionContext_t dctx,
+                         LZ4F_frameInfo_t* frameInfoPtr,
+                         const void* srcBuffer, size_t* srcSizePtr);
+/* LZ4F_getFrameInfo()
+ * This function decodes frame header information (such as max blockSize, frame checksum, etc.).
+ * Its usage is optional : you can start by calling directly LZ4F_decompress() instead.
+ * The objective is to extract frame header information, typically for allocation purposes.
+ * LZ4F_getFrameInfo() can also be used anytime *after* starting decompression, on any valid LZ4F_decompressionContext_t.
+ * The result is *copied* into an existing LZ4F_frameInfo_t structure which must be already allocated.
+ * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call,
+ *                        or an error code which can be tested using LZ4F_isError()
+ *                        (typically, when there is not enough src bytes to fully decode the frame header)
+ * You are expected to resume decompression from where it stopped (srcBuffer + *srcSizePtr)
+ */
+
+size_t LZ4F_decompress(LZ4F_decompressionContext_t dctx,
+                       void* dstBuffer, size_t* dstSizePtr,
+                       const void* srcBuffer, size_t* srcSizePtr,
+                       const LZ4F_decompressOptions_t* dOptPtr);
+/* LZ4F_decompress()
+ * Call this function repetitively to regenerate data compressed within srcBuffer.
+ * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
+ *
+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ *
+ * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * If number of bytes read is < number of bytes provided, then decompression operation is not completed.
+ * It typically happens when dstBuffer is not large enough to contain all decoded data.
+ * LZ4F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr)
+ * The function will check this condition, and refuse to continue if it is not respected.
+ *
+ * dstBuffer is supposed to be flushed between each call to the function, since its content will be overwritten.
+ * dst arguments can be changed at will with each consecutive call to the function.
+ *
+ * The function result is an hint of how many srcSize bytes LZ4F_decompress() expects for next call.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
+ * This is just a hint, you can always provide any srcSize you want.
+ * When a frame is fully decoded, the function result will be 0 (no more data expected).
+ * If decompression failed, function result is an error code, which can be tested using LZ4F_isError().
+ *
+ * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ */
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz4/lz4frame_static.h

@@ -0,0 +1,81 @@
+/*
+   LZ4 auto-framing library
+   Header File for static linking only
+   Copyright (C) 2011-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* lz4frame_static.h should be used solely in the context of static linking.
+ * It contains definitions which may still change overtime.
+ * Never use it in the context of DLL linking.
+ * */
+
+
+/**************************************
+*  Includes
+**************************************/
+#include "lz4frame.h"
+
+
+/**************************************
+ * Error management
+ * ************************************/
+#define LZ4F_LIST_ERRORS(ITEM) \
+        ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
+        ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
+        ITEM(ERROR_compressionLevel_invalid) \
+        ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
+        ITEM(ERROR_allocation_failed) \
+        ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
+        ITEM(ERROR_srcPtr_wrong) \
+        ITEM(ERROR_decompressionFailed) \
+        ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
+        ITEM(ERROR_maxCode)
+
+//#define LZ4F_DISABLE_OLD_ENUMS
+#ifndef LZ4F_DISABLE_OLD_ENUMS
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM, ENUM = LZ4F_##ENUM,
+#else
+#define LZ4F_GENERATE_ENUM(ENUM) LZ4F_##ENUM,
+#endif
+typedef enum { LZ4F_LIST_ERRORS(LZ4F_GENERATE_ENUM) } LZ4F_errorCodes;  /* enum is exposed, to handle specific errors; compare function result to -enum value */
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz4/lz4hc.c

@@ -0,0 +1,731 @@
+/*
+    LZ4 HC - High Compression Mode of LZ4
+    Copyright (C) 2011-2015, Yann Collet.
+
+    BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+    Redistribution and use in source and binary forms, with or without
+    modification, are permitted provided that the following conditions are
+    met:
+
+    * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+    copyright notice, this list of conditions and the following disclaimer
+    in the documentation and/or other materials provided with the
+    distribution.
+
+    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+    A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+    OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+    SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+    DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+    THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+    (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+    OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+       - LZ4 source repository : https://github.com/Cyan4973/lz4
+       - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+
+
+
+/**************************************
+*  Tuning Parameter
+**************************************/
+static const int LZ4HC_compressionLevel_default = 9;
+
+
+/**************************************
+*  Includes
+**************************************/
+#include "lz4hc.h"
+
+
+/**************************************
+*  Local Compiler Options
+**************************************/
+#if defined(__GNUC__)
+#  pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+
+#if defined (__clang__)
+#  pragma clang diagnostic ignored "-Wunused-function"
+#endif
+
+
+/**************************************
+*  Common LZ4 definition
+**************************************/
+#define LZ4_COMMONDEFS_ONLY
+#include "lz4.c"
+
+
+/**************************************
+*  Local Constants
+**************************************/
+#define DICTIONARY_LOGSIZE 16
+#define MAXD (1<<DICTIONARY_LOGSIZE)
+#define MAXD_MASK (MAXD - 1)
+
+#define HASH_LOG (DICTIONARY_LOGSIZE-1)
+#define HASHTABLESIZE (1 << HASH_LOG)
+#define HASH_MASK (HASHTABLESIZE - 1)
+
+#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
+
+static const int g_maxCompressionLevel = 16;
+
+
+/**************************************
+*  Local Types
+**************************************/
+typedef struct
+{
+    U32   hashTable[HASHTABLESIZE];
+    U16   chainTable[MAXD];
+    const BYTE* end;        /* next block here to continue on current prefix */
+    const BYTE* base;       /* All index relative to this position */
+    const BYTE* dictBase;   /* alternate base for extDict */
+    BYTE* inputBuffer;      /* deprecated */
+    U32   dictLimit;        /* below that point, need extDict */
+    U32   lowLimit;         /* below that point, no more dict */
+    U32   nextToUpdate;     /* index from which to continue dictionary update */
+    U32   compressionLevel;
+} LZ4HC_Data_Structure;
+
+
+/**************************************
+*  Local Macros
+**************************************/
+#define HASH_FUNCTION(i)       (((i) * 2654435761U) >> ((MINMATCH*8)-HASH_LOG))
+//#define DELTANEXTU16(p)        chainTable[(p) & MAXD_MASK]   /* flexible, MAXD dependent */
+#define DELTANEXTU16(p)        chainTable[(U16)(p)]   /* faster */
+
+static U32 LZ4HC_hashPtr(const void* ptr) { return HASH_FUNCTION(LZ4_read32(ptr)); }
+
+
+
+/**************************************
+*  HC Compression
+**************************************/
+static void LZ4HC_init (LZ4HC_Data_Structure* hc4, const BYTE* start)
+{
+    MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable));
+    MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable));
+    hc4->nextToUpdate = 64 KB;
+    hc4->base = start - 64 KB;
+    hc4->end = start;
+    hc4->dictBase = start - 64 KB;
+    hc4->dictLimit = 64 KB;
+    hc4->lowLimit = 64 KB;
+}
+
+
+/* Update chains up to ip (excluded) */
+FORCE_INLINE void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip)
+{
+    U16* chainTable = hc4->chainTable;
+    U32* HashTable  = hc4->hashTable;
+    const BYTE* const base = hc4->base;
+    const U32 target = (U32)(ip - base);
+    U32 idx = hc4->nextToUpdate;
+
+    while(idx < target)
+    {
+        U32 h = LZ4HC_hashPtr(base+idx);
+        size_t delta = idx - HashTable[h];
+        if (delta>MAX_DISTANCE) delta = MAX_DISTANCE;
+        DELTANEXTU16(idx) = (U16)delta;
+        HashTable[h] = idx;
+        idx++;
+    }
+
+    hc4->nextToUpdate = target;
+}
+
+
+FORCE_INLINE int LZ4HC_InsertAndFindBestMatch (LZ4HC_Data_Structure* hc4,   /* Index table will be updated */
+                                               const BYTE* ip, const BYTE* const iLimit,
+                                               const BYTE** matchpos,
+                                               const int maxNbAttempts)
+{
+    U16* const chainTable = hc4->chainTable;
+    U32* const HashTable = hc4->hashTable;
+    const BYTE* const base = hc4->base;
+    const BYTE* const dictBase = hc4->dictBase;
+    const U32 dictLimit = hc4->dictLimit;
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
+    U32 matchIndex;
+    const BYTE* match;
+    int nbAttempts=maxNbAttempts;
+    size_t ml=0;
+
+    /* HC4 match finder */
+    LZ4HC_Insert(hc4, ip);
+    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+
+    while ((matchIndex>=lowLimit) && (nbAttempts))
+    {
+        nbAttempts--;
+        if (matchIndex >= dictLimit)
+        {
+            match = base + matchIndex;
+            if (*(match+ml) == *(ip+ml)
+                && (LZ4_read32(match) == LZ4_read32(ip)))
+            {
+                size_t mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, iLimit) + MINMATCH;
+                if (mlt > ml) { ml = mlt; *matchpos = match; }
+            }
+        }
+        else
+        {
+            match = dictBase + matchIndex;
+            if (LZ4_read32(match) == LZ4_read32(ip))
+            {
+                size_t mlt;
+                const BYTE* vLimit = ip + (dictLimit - matchIndex);
+                if (vLimit > iLimit) vLimit = iLimit;
+                mlt = LZ4_count(ip+MINMATCH, match+MINMATCH, vLimit) + MINMATCH;
+                if ((ip+mlt == vLimit) && (vLimit < iLimit))
+                    mlt += LZ4_count(ip+mlt, base+dictLimit, iLimit);
+                if (mlt > ml) { ml = mlt; *matchpos = base + matchIndex; }   /* virtual matchpos */
+            }
+        }
+        matchIndex -= DELTANEXTU16(matchIndex);
+    }
+
+    return (int)ml;
+}
+
+
+FORCE_INLINE int LZ4HC_InsertAndGetWiderMatch (
+    LZ4HC_Data_Structure* hc4,
+    const BYTE* const ip,
+    const BYTE* const iLowLimit,
+    const BYTE* const iHighLimit,
+    int longest,
+    const BYTE** matchpos,
+    const BYTE** startpos,
+    const int maxNbAttempts)
+{
+    U16* const chainTable = hc4->chainTable;
+    U32* const HashTable = hc4->hashTable;
+    const BYTE* const base = hc4->base;
+    const U32 dictLimit = hc4->dictLimit;
+    const BYTE* const lowPrefixPtr = base + dictLimit;
+    const U32 lowLimit = (hc4->lowLimit + 64 KB > (U32)(ip-base)) ? hc4->lowLimit : (U32)(ip - base) - (64 KB - 1);
+    const BYTE* const dictBase = hc4->dictBase;
+    U32   matchIndex;
+    int nbAttempts = maxNbAttempts;
+    int delta = (int)(ip-iLowLimit);
+
+
+    /* First Match */
+    LZ4HC_Insert(hc4, ip);
+    matchIndex = HashTable[LZ4HC_hashPtr(ip)];
+
+    while ((matchIndex>=lowLimit) && (nbAttempts))
+    {
+        nbAttempts--;
+        if (matchIndex >= dictLimit)
+        {
+            const BYTE* matchPtr = base + matchIndex;
+            if (*(iLowLimit + longest) == *(matchPtr - delta + longest))
+                if (LZ4_read32(matchPtr) == LZ4_read32(ip))
+                {
+                    int mlt = MINMATCH + LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, iHighLimit);
+                    int back = 0;
+
+                    while ((ip+back>iLowLimit)
+                           && (matchPtr+back > lowPrefixPtr)
+                           && (ip[back-1] == matchPtr[back-1]))
+                            back--;
+
+                    mlt -= back;
+
+                    if (mlt > longest)
+                    {
+                        longest = (int)mlt;
+                        *matchpos = matchPtr+back;
+                        *startpos = ip+back;
+                    }
+                }
+        }
+        else
+        {
+            const BYTE* matchPtr = dictBase + matchIndex;
+            if (LZ4_read32(matchPtr) == LZ4_read32(ip))
+            {
+                size_t mlt;
+                int back=0;
+                const BYTE* vLimit = ip + (dictLimit - matchIndex);
+                if (vLimit > iHighLimit) vLimit = iHighLimit;
+                mlt = LZ4_count(ip+MINMATCH, matchPtr+MINMATCH, vLimit) + MINMATCH;
+                if ((ip+mlt == vLimit) && (vLimit < iHighLimit))
+                    mlt += LZ4_count(ip+mlt, base+dictLimit, iHighLimit);
+                while ((ip+back > iLowLimit) && (matchIndex+back > lowLimit) && (ip[back-1] == matchPtr[back-1])) back--;
+                mlt -= back;
+                if ((int)mlt > longest) { longest = (int)mlt; *matchpos = base + matchIndex + back; *startpos = ip+back; }
+            }
+        }
+        matchIndex -= DELTANEXTU16(matchIndex);
+    }
+
+    return longest;
+}
+
+
+typedef enum { noLimit = 0, limitedOutput = 1 } limitedOutput_directive;
+
+#define LZ4HC_DEBUG 0
+#if LZ4HC_DEBUG
+static unsigned debug = 0;
+#endif
+
+FORCE_INLINE int LZ4HC_encodeSequence (
+    const BYTE** ip,
+    BYTE** op,
+    const BYTE** anchor,
+    int matchLength,
+    const BYTE* const match,
+    limitedOutput_directive limitedOutputBuffer,
+    BYTE* oend)
+{
+    int length;
+    BYTE* token;
+
+#if LZ4HC_DEBUG
+    if (debug) printf("literal : %u  --  match : %u  --  offset : %u\n", (U32)(*ip - *anchor), (U32)matchLength, (U32)(*ip-match));
+#endif
+
+    /* Encode Literal length */
+    length = (int)(*ip - *anchor);
+    token = (*op)++;
+    if ((limitedOutputBuffer) && ((*op + (length>>8) + length + (2 + 1 + LASTLITERALS)) > oend)) return 1;   /* Check output limit */
+    if (length>=(int)RUN_MASK) { int len; *token=(RUN_MASK<<ML_BITS); len = length-RUN_MASK; for(; len > 254 ; len-=255) *(*op)++ = 255;  *(*op)++ = (BYTE)len; }
+    else *token = (BYTE)(length<<ML_BITS);
+
+    /* Copy Literals */
+    LZ4_wildCopy(*op, *anchor, (*op) + length);
+    *op += length;
+
+    /* Encode Offset */
+    LZ4_writeLE16(*op, (U16)(*ip-match)); *op += 2;
+
+    /* Encode MatchLength */
+    length = (int)(matchLength-MINMATCH);
+    if ((limitedOutputBuffer) && (*op + (length>>8) + (1 + LASTLITERALS) > oend)) return 1;   /* Check output limit */
+    if (length>=(int)ML_MASK) { *token+=ML_MASK; length-=ML_MASK; for(; length > 509 ; length-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (length > 254) { length-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)length; }
+    else *token += (BYTE)(length);
+
+    /* Prepare next loop */
+    *ip += matchLength;
+    *anchor = *ip;
+
+    return 0;
+}
+
+
+static int LZ4HC_compress_generic (
+    void* ctxvoid,
+    const char* source,
+    char* dest,
+    int inputSize,
+    int maxOutputSize,
+    int compressionLevel,
+    limitedOutput_directive limit
+    )
+{
+    LZ4HC_Data_Structure* ctx = (LZ4HC_Data_Structure*) ctxvoid;
+    const BYTE* ip = (const BYTE*) source;
+    const BYTE* anchor = ip;
+    const BYTE* const iend = ip + inputSize;
+    const BYTE* const mflimit = iend - MFLIMIT;
+    const BYTE* const matchlimit = (iend - LASTLITERALS);
+
+    BYTE* op = (BYTE*) dest;
+    BYTE* const oend = op + maxOutputSize;
+
+    unsigned maxNbAttempts;
+    int   ml, ml2, ml3, ml0;
+    const BYTE* ref=NULL;
+    const BYTE* start2=NULL;
+    const BYTE* ref2=NULL;
+    const BYTE* start3=NULL;
+    const BYTE* ref3=NULL;
+    const BYTE* start0;
+    const BYTE* ref0;
+
+
+    /* init */
+    if (compressionLevel > g_maxCompressionLevel) compressionLevel = g_maxCompressionLevel;
+    if (compressionLevel < 1) compressionLevel = LZ4HC_compressionLevel_default;
+    maxNbAttempts = 1 << (compressionLevel-1);
+    ctx->end += inputSize;
+
+    ip++;
+
+    /* Main Loop */
+    while (ip < mflimit)
+    {
+        ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref), maxNbAttempts);
+        if (!ml) { ip++; continue; }
+
+        /* saved, in case we would skip too much */
+        start0 = ip;
+        ref0 = ref;
+        ml0 = ml;
+
+_Search2:
+        if (ip+ml < mflimit)
+            ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2, maxNbAttempts);
+        else ml2 = ml;
+
+        if (ml2 == ml)  /* No better match */
+        {
+            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+            continue;
+        }
+
+        if (start0 < ip)
+        {
+            if (start2 < ip + ml0)   /* empirical */
+            {
+                ip = start0;
+                ref = ref0;
+                ml = ml0;
+            }
+        }
+
+        /* Here, start0==ip */
+        if ((start2 - ip) < 3)   /* First Match too small : removed */
+        {
+            ml = ml2;
+            ip = start2;
+            ref =ref2;
+            goto _Search2;
+        }
+
+_Search3:
+        /*
+        * Currently we have :
+        * ml2 > ml1, and
+        * ip1+3 <= ip2 (usually < ip1+ml1)
+        */
+        if ((start2 - ip) < OPTIMAL_ML)
+        {
+            int correction;
+            int new_ml = ml;
+            if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML;
+            if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
+            correction = new_ml - (int)(start2 - ip);
+            if (correction > 0)
+            {
+                start2 += correction;
+                ref2 += correction;
+                ml2 -= correction;
+            }
+        }
+        /* Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) */
+
+        if (start2 + ml2 < mflimit)
+            ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3, maxNbAttempts);
+        else ml3 = ml2;
+
+        if (ml3 == ml2) /* No better match : 2 sequences to encode */
+        {
+            /* ip & ref are known; Now for ml */
+            if (start2 < ip+ml)  ml = (int)(start2 - ip);
+            /* Now, encode 2 sequences */
+            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+            ip = start2;
+            if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml2, ref2, limit, oend)) return 0;
+            continue;
+        }
+
+        if (start3 < ip+ml+3) /* Not enough space for match 2 : remove it */
+        {
+            if (start3 >= (ip+ml)) /* can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 */
+            {
+                if (start2 < ip+ml)
+                {
+                    int correction = (int)(ip+ml - start2);
+                    start2 += correction;
+                    ref2 += correction;
+                    ml2 -= correction;
+                    if (ml2 < MINMATCH)
+                    {
+                        start2 = start3;
+                        ref2 = ref3;
+                        ml2 = ml3;
+                    }
+                }
+
+                if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+                ip  = start3;
+                ref = ref3;
+                ml  = ml3;
+
+                start0 = start2;
+                ref0 = ref2;
+                ml0 = ml2;
+                goto _Search2;
+            }
+
+            start2 = start3;
+            ref2 = ref3;
+            ml2 = ml3;
+            goto _Search3;
+        }
+
+        /*
+        * OK, now we have 3 ascending matches; let's write at least the first one
+        * ip & ref are known; Now for ml
+        */
+        if (start2 < ip+ml)
+        {
+            if ((start2 - ip) < (int)ML_MASK)
+            {
+                int correction;
+                if (ml > OPTIMAL_ML) ml = OPTIMAL_ML;
+                if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH;
+                correction = ml - (int)(start2 - ip);
+                if (correction > 0)
+                {
+                    start2 += correction;
+                    ref2 += correction;
+                    ml2 -= correction;
+                }
+            }
+            else
+            {
+                ml = (int)(start2 - ip);
+            }
+        }
+        if (LZ4HC_encodeSequence(&ip, &op, &anchor, ml, ref, limit, oend)) return 0;
+
+        ip = start2;
+        ref = ref2;
+        ml = ml2;
+
+        start2 = start3;
+        ref2 = ref3;
+        ml2 = ml3;
+
+        goto _Search3;
+    }
+
+    /* Encode Last Literals */
+    {
+        int lastRun = (int)(iend - anchor);
+        if ((limit) && (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize)) return 0;  /* Check output limit */
+        if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK<<ML_BITS); lastRun-=RUN_MASK; for(; lastRun > 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; }
+        else *op++ = (BYTE)(lastRun<<ML_BITS);
+        memcpy(op, anchor, iend - anchor);
+        op += iend-anchor;
+    }
+
+    /* End */
+    return (int) (((char*)op)-dest);
+}
+
+
+int LZ4_sizeofStateHC(void) { return sizeof(LZ4HC_Data_Structure); }
+
+int LZ4_compress_HC_extStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
+{
+    if (((size_t)(state)&(sizeof(void*)-1)) != 0) return 0;   /* Error : state is not aligned for pointers (32 or 64 bits) */
+    LZ4HC_init ((LZ4HC_Data_Structure*)state, (const BYTE*)src);
+    if (maxDstSize < LZ4_compressBound(srcSize))
+        return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, limitedOutput);
+    else
+        return LZ4HC_compress_generic (state, src, dst, srcSize, maxDstSize, compressionLevel, noLimit);
+}
+
+int LZ4_compress_HC(const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel)
+{
+    LZ4HC_Data_Structure state;
+    return LZ4_compress_HC_extStateHC(&state, src, dst, srcSize, maxDstSize, compressionLevel);
+}
+
+
+
+/**************************************
+*  Streaming Functions
+**************************************/
+/* allocation */
+LZ4_streamHC_t* LZ4_createStreamHC(void) { return (LZ4_streamHC_t*)malloc(sizeof(LZ4_streamHC_t)); }
+int             LZ4_freeStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr) { free(LZ4_streamHCPtr); return 0; }
+
+
+/* initialization */
+void LZ4_resetStreamHC (LZ4_streamHC_t* LZ4_streamHCPtr, int compressionLevel)
+{
+    LZ4_STATIC_ASSERT(sizeof(LZ4HC_Data_Structure) <= sizeof(LZ4_streamHC_t));   /* if compilation fails here, LZ4_STREAMHCSIZE must be increased */
+    ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->base = NULL;
+    ((LZ4HC_Data_Structure*)LZ4_streamHCPtr)->compressionLevel = (unsigned)compressionLevel;
+}
+
+int LZ4_loadDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, const char* dictionary, int dictSize)
+{
+    LZ4HC_Data_Structure* ctxPtr = (LZ4HC_Data_Structure*) LZ4_streamHCPtr;
+    if (dictSize > 64 KB)
+    {
+        dictionary += dictSize - 64 KB;
+        dictSize = 64 KB;
+    }
+    LZ4HC_init (ctxPtr, (const BYTE*)dictionary);
+    if (dictSize >= 4) LZ4HC_Insert (ctxPtr, (const BYTE*)dictionary +(dictSize-3));
+    ctxPtr->end = (const BYTE*)dictionary + dictSize;
+    return dictSize;
+}
+
+
+/* compression */
+
+static void LZ4HC_setExternalDict(LZ4HC_Data_Structure* ctxPtr, const BYTE* newBlock)
+{
+    if (ctxPtr->end >= ctxPtr->base + 4)
+        LZ4HC_Insert (ctxPtr, ctxPtr->end-3);   /* Referencing remaining dictionary content */
+    /* Only one memory segment for extDict, so any previous extDict is lost at this stage */
+    ctxPtr->lowLimit  = ctxPtr->dictLimit;
+    ctxPtr->dictLimit = (U32)(ctxPtr->end - ctxPtr->base);
+    ctxPtr->dictBase  = ctxPtr->base;
+    ctxPtr->base = newBlock - ctxPtr->dictLimit;
+    ctxPtr->end  = newBlock;
+    ctxPtr->nextToUpdate = ctxPtr->dictLimit;   /* match referencing will resume from there */
+}
+
+static int LZ4_compressHC_continue_generic (LZ4HC_Data_Structure* ctxPtr,
+                                            const char* source, char* dest,
+                                            int inputSize, int maxOutputSize, limitedOutput_directive limit)
+{
+    /* auto-init if forgotten */
+    if (ctxPtr->base == NULL)
+        LZ4HC_init (ctxPtr, (const BYTE*) source);
+
+    /* Check overflow */
+    if ((size_t)(ctxPtr->end - ctxPtr->base) > 2 GB)
+    {
+        size_t dictSize = (size_t)(ctxPtr->end - ctxPtr->base) - ctxPtr->dictLimit;
+        if (dictSize > 64 KB) dictSize = 64 KB;
+
+        LZ4_loadDictHC((LZ4_streamHC_t*)ctxPtr, (const char*)(ctxPtr->end) - dictSize, (int)dictSize);
+    }
+
+    /* Check if blocks follow each other */
+    if ((const BYTE*)source != ctxPtr->end)
+        LZ4HC_setExternalDict(ctxPtr, (const BYTE*)source);
+
+    /* Check overlapping input/dictionary space */
+    {
+        const BYTE* sourceEnd = (const BYTE*) source + inputSize;
+        const BYTE* dictBegin = ctxPtr->dictBase + ctxPtr->lowLimit;
+        const BYTE* dictEnd   = ctxPtr->dictBase + ctxPtr->dictLimit;
+        if ((sourceEnd > dictBegin) && ((const BYTE*)source < dictEnd))
+        {
+            if (sourceEnd > dictEnd) sourceEnd = dictEnd;
+            ctxPtr->lowLimit = (U32)(sourceEnd - ctxPtr->dictBase);
+            if (ctxPtr->dictLimit - ctxPtr->lowLimit < 4) ctxPtr->lowLimit = ctxPtr->dictLimit;
+        }
+    }
+
+    return LZ4HC_compress_generic (ctxPtr, source, dest, inputSize, maxOutputSize, ctxPtr->compressionLevel, limit);
+}
+
+int LZ4_compress_HC_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize)
+{
+    if (maxOutputSize < LZ4_compressBound(inputSize))
+        return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, limitedOutput);
+    else
+        return LZ4_compressHC_continue_generic ((LZ4HC_Data_Structure*)LZ4_streamHCPtr, source, dest, inputSize, maxOutputSize, noLimit);
+}
+
+
+/* dictionary saving */
+
+int LZ4_saveDictHC (LZ4_streamHC_t* LZ4_streamHCPtr, char* safeBuffer, int dictSize)
+{
+    LZ4HC_Data_Structure* streamPtr = (LZ4HC_Data_Structure*)LZ4_streamHCPtr;
+    int prefixSize = (int)(streamPtr->end - (streamPtr->base + streamPtr->dictLimit));
+    if (dictSize > 64 KB) dictSize = 64 KB;
+    if (dictSize < 4) dictSize = 0;
+    if (dictSize > prefixSize) dictSize = prefixSize;
+    memmove(safeBuffer, streamPtr->end - dictSize, dictSize);
+    {
+        U32 endIndex = (U32)(streamPtr->end - streamPtr->base);
+        streamPtr->end = (const BYTE*)safeBuffer + dictSize;
+        streamPtr->base = streamPtr->end - endIndex;
+        streamPtr->dictLimit = endIndex - dictSize;
+        streamPtr->lowLimit = endIndex - dictSize;
+        if (streamPtr->nextToUpdate < streamPtr->dictLimit) streamPtr->nextToUpdate = streamPtr->dictLimit;
+    }
+    return dictSize;
+}
+
+
+/***********************************
+*  Deprecated Functions
+***********************************/
+/* Deprecated compression functions */
+/* These functions are planned to start generate warnings by r131 approximately */
+int LZ4_compressHC(const char* src, char* dst, int srcSize) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
+int LZ4_compressHC_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, 0); }
+int LZ4_compressHC2(const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC (src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
+int LZ4_compressHC2_limitedOutput(const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC(src, dst, srcSize, maxDstSize, cLevel); }
+int LZ4_compressHC_withStateHC (void* state, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, LZ4_compressBound(srcSize), 0); }
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_extStateHC (state, src, dst, srcSize, maxDstSize, 0); }
+int LZ4_compressHC2_withStateHC (void* state, const char* src, char* dst, int srcSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, LZ4_compressBound(srcSize), cLevel); }
+int LZ4_compressHC2_limitedOutput_withStateHC (void* state, const char* src, char* dst, int srcSize, int maxDstSize, int cLevel) { return LZ4_compress_HC_extStateHC(state, src, dst, srcSize, maxDstSize, cLevel); }
+int LZ4_compressHC_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, LZ4_compressBound(srcSize)); }
+int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* ctx, const char* src, char* dst, int srcSize, int maxDstSize) { return LZ4_compress_HC_continue (ctx, src, dst, srcSize, maxDstSize); }
+
+
+/* Deprecated streaming functions */
+/* These functions currently generate deprecation warnings */
+int LZ4_sizeofStreamStateHC(void) { return LZ4_STREAMHCSIZE; }
+
+int LZ4_resetStreamStateHC(void* state, char* inputBuffer)
+{
+    if ((((size_t)state) & (sizeof(void*)-1)) != 0) return 1;   /* Error : pointer is not aligned for pointer (32 or 64 bits) */
+    LZ4HC_init((LZ4HC_Data_Structure*)state, (const BYTE*)inputBuffer);
+    ((LZ4HC_Data_Structure*)state)->inputBuffer = (BYTE*)inputBuffer;
+    return 0;
+}
+
+void* LZ4_createHC (char* inputBuffer)
+{
+    void* hc4 = ALLOCATOR(1, sizeof(LZ4HC_Data_Structure));
+    if (hc4 == NULL) return NULL;   /* not enough memory */
+    LZ4HC_init ((LZ4HC_Data_Structure*)hc4, (const BYTE*)inputBuffer);
+    ((LZ4HC_Data_Structure*)hc4)->inputBuffer = (BYTE*)inputBuffer;
+    return hc4;
+}
+
+int LZ4_freeHC (void* LZ4HC_Data)
+{
+    FREEMEM(LZ4HC_Data);
+    return (0);
+}
+
+int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel)
+{
+    return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, 0, compressionLevel, noLimit);
+}
+
+int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel)
+{
+    return LZ4HC_compress_generic (LZ4HC_Data, source, dest, inputSize, maxOutputSize, compressionLevel, limitedOutput);
+}
+
+char* LZ4_slideInputBufferHC(void* LZ4HC_Data)
+{
+    LZ4HC_Data_Structure* hc4 = (LZ4HC_Data_Structure*)LZ4HC_Data;
+    int dictSize = LZ4_saveDictHC((LZ4_streamHC_t*)LZ4HC_Data, (char*)(hc4->inputBuffer), 64 KB);
+    return (char*)(hc4->inputBuffer + dictSize);
+}

C/lz4/lz4hc.h

@@ -0,0 +1,189 @@
+/*
+   LZ4 HC - High Compression Mode of LZ4
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ4 source repository : https://github.com/Cyan4973/lz4
+   - LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c
+*/
+#pragma once
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*****************************
+*  Includes
+*****************************/
+#include <stddef.h>   /* size_t */
+
+
+/**************************************
+*  Block Compression
+**************************************/
+int LZ4_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
+/*
+LZ4_compress_HC :
+    Destination buffer 'dst' must be already allocated.
+    Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible)
+    Worst size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
+      srcSize  : Max supported value is LZ4_MAX_INPUT_SIZE (see "lz4.h")
+      compressionLevel : Recommended values are between 4 and 9, although any value between 0 and 16 will work.
+                         0 means "use default value" (see lz4hc.c).
+                         Values >16 behave the same as 16.
+      return : the number of bytes written into buffer 'dst'
+            or 0 if compression fails.
+*/
+
+
+/* Note :
+   Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
+*/
+
+
+int LZ4_sizeofStateHC(void);
+int LZ4_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
+/*
+LZ4_compress_HC_extStateHC() :
+   Use this function if you prefer to manually allocate memory for compression tables.
+   To know how much memory must be allocated for the compression tables, use :
+      int LZ4_sizeofStateHC();
+
+   Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
+
+   The allocated memory can then be provided to the compression functions using 'void* state' parameter.
+   LZ4_compress_HC_extStateHC() is equivalent to previously described function.
+   It just uses externally allocated memory for stateHC.
+*/
+
+
+/**************************************
+*  Streaming Compression
+**************************************/
+#define LZ4_STREAMHCSIZE        262192
+#define LZ4_STREAMHCSIZE_SIZET (LZ4_STREAMHCSIZE / sizeof(size_t))
+typedef struct { size_t table[LZ4_STREAMHCSIZE_SIZET]; } LZ4_streamHC_t;
+/*
+  LZ4_streamHC_t
+  This structure allows static allocation of LZ4 HC streaming state.
+  State must then be initialized using LZ4_resetStreamHC() before first use.
+
+  Static allocation should only be used in combination with static linking.
+  If you want to use LZ4 as a DLL, please use construction functions below, which are future-proof.
+*/
+
+
+LZ4_streamHC_t* LZ4_createStreamHC(void);
+int             LZ4_freeStreamHC (LZ4_streamHC_t* streamHCPtr);
+/*
+  These functions create and release memory for LZ4 HC streaming state.
+  Newly created states are already initialized.
+  Existing state space can be re-used anytime using LZ4_resetStreamHC().
+  If you use LZ4 as a DLL, use these functions instead of static structure allocation,
+  to avoid size mismatch between different versions.
+*/
+
+void LZ4_resetStreamHC (LZ4_streamHC_t* streamHCPtr, int compressionLevel);
+int  LZ4_loadDictHC (LZ4_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
+
+int LZ4_compress_HC_continue (LZ4_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
+
+int LZ4_saveDictHC (LZ4_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
+
+/*
+  These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+  One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
+  There is an exception for ring buffers, which can be smaller 64 KB.
+  Such case is automatically detected and correctly handled by LZ4_compress_HC_continue().
+
+  Before starting compression, state must be properly initialized, using LZ4_resetStreamHC().
+  A first "fictional block" can then be designated as initial dictionary, using LZ4_loadDictHC() (Optional).
+
+  Then, use LZ4_compress_HC_continue() to compress each successive block.
+  It works like LZ4_compress_HC(), but use previous memory blocks as dictionary to improve compression.
+  Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
+  As a reminder, size 'dst' buffer to handle worst cases, using LZ4_compressBound(), to ensure success of compression operation.
+
+  If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
+  you must save it to a safer memory space, using LZ4_saveDictHC().
+  Return value of LZ4_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
+*/
+
+
+
+/**************************************
+*  Deprecated Functions
+**************************************/
+/* Deprecate Warnings */
+/* Should these warnings messages be a problem,
+   it is generally possible to disable them,
+   with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual for example.
+   You can also define LZ4_DEPRECATE_WARNING_DEFBLOCK. */
+#ifndef LZ4_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ4_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ4_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#  if (LZ4_GCC_VERSION >= 405) || defined(__clang__)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (LZ4_GCC_VERSION >= 301)
+#    define LZ4_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define LZ4_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement LZ4_DEPRECATED for this compiler")
+#    define LZ4_DEPRECATED(message)
+#  endif
+#endif // LZ4_DEPRECATE_WARNING_DEFBLOCK
+
+/* compression functions */
+/* these functions are planned to trigger warning messages by r131 approximately */
+int LZ4_compressHC                (const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput  (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compressHC2               (const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+int LZ4_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ4_compressHC2_withStateHC              (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
+int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+int LZ4_compressHC_continue               (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize);
+int LZ4_compressHC_limitedOutput_continue (LZ4_streamHC_t* LZ4_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Streaming functions following the older model; should no longer be used */
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") void* LZ4_createHC (char* inputBuffer);
+LZ4_DEPRECATED("use LZ4_saveDictHC() instead")     char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_freeStreamHC() instead")   int   LZ4_freeHC (void* LZ4HC_Data);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int   LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_compress_HC_continue() instead") int   LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
+LZ4_DEPRECATED("use LZ4_createStreamHC() instead") int   LZ4_sizeofStreamStateHC(void);
+LZ4_DEPRECATED("use LZ4_resetStreamHC() instead")  int   LZ4_resetStreamStateHC(void* state, char* inputBuffer);
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz4/lz4xxhash.c

@@ -0,0 +1,915 @@
+/*
+xxHash - Fast Hash algorithm
+Copyright (C) 2012-2015, Yann Collet
+
+BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/**************************************
+*  Tuning parameters
+**************************************/
+/* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+ * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
+ * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
+ * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
+ */
+#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#  define XXH_USE_UNALIGNED_ACCESS 1
+#endif
+
+/* XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+ * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+ * By default, this option is disabled. To enable it, uncomment below define :
+ */
+/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+
+/* XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independance be of no importance for your application, you may set the #define below to 1.
+ * It will improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#define XXH_FORCE_NATIVE_FORMAT 0
+
+
+/**************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
+#  define FORCE_INLINE static __forceinline
+#else
+#  if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
+#endif
+
+
+/**************************************
+*  Includes & Memory related functions
+***************************************/
+#include "xxhash.h"
+/* Modify the local functions below should you wish to use some other memory routines */
+/* for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void  XXH_free  (void* p)  { free(p); }
+/* for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+
+/**************************************
+*  Basic Types
+***************************************/
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+# include <stdint.h>
+  typedef uint8_t  BYTE;
+  typedef uint16_t U16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+#else
+  typedef unsigned char      BYTE;
+  typedef unsigned short     U16;
+  typedef unsigned int       U32;
+  typedef   signed int       S32;
+  typedef unsigned long long U64;
+#endif
+
+static U32 XXH_read32(const void* memPtr)
+{
+    U32 val32;
+    memcpy(&val32, memPtr, 4);
+    return val32;
+}
+
+static U64 XXH_read64(const void* memPtr)
+{
+    U64 val64;
+    memcpy(&val64, memPtr, 8);
+    return val64;
+}
+
+
+
+/******************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#  define XXH_swap64 _byteswap_uint64
+#elif GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#  define XXH_swap64 __builtin_bswap64
+#else
+static U32 XXH_swap32 (U32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+static U64 XXH_swap64 (U64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+
+/***************************************
+*  Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+#ifndef XXH_CPU_LITTLE_ENDIAN   /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */
+static const int one = 1;
+#   define XXH_CPU_LITTLE_ENDIAN   (*(const char*)(&one))
+#endif
+
+
+/*****************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+
+/***************************************
+*  Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(!!(c)) }; }    /* use only *after* variable declarations */
+
+
+/***************************************
+*  Constants
+***************************************/
+#define PRIME32_1   2654435761U
+#define PRIME32_2   2246822519U
+#define PRIME32_3   3266489917U
+#define PRIME32_4    668265263U
+#define PRIME32_5    374761393U
+
+#define PRIME64_1 11400714785074694791ULL
+#define PRIME64_2 14029467366897019727ULL
+#define PRIME64_3  1609587929392839161ULL
+#define PRIME64_4  9650029242287828579ULL
+#define PRIME64_5  2870177450012600261ULL
+
+
+/*****************************
+*  Simple Hash Functions
+*****************************/
+FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U32 h32;
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL)
+    {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)16;
+    }
+#endif
+
+    if (len>=16)
+    {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = seed + PRIME32_1 + PRIME32_2;
+        U32 v2 = seed + PRIME32_2;
+        U32 v3 = seed + 0;
+        U32 v4 = seed - PRIME32_1;
+
+        do
+        {
+            v1 += XXH_get32bits(p) * PRIME32_2;
+            v1 = XXH_rotl32(v1, 13);
+            v1 *= PRIME32_1;
+            p+=4;
+            v2 += XXH_get32bits(p) * PRIME32_2;
+            v2 = XXH_rotl32(v2, 13);
+            v2 *= PRIME32_1;
+            p+=4;
+            v3 += XXH_get32bits(p) * PRIME32_2;
+            v3 = XXH_rotl32(v3, 13);
+            v3 *= PRIME32_1;
+            p+=4;
+            v4 += XXH_get32bits(p) * PRIME32_2;
+            v4 = XXH_rotl32(v4, 13);
+            v4 *= PRIME32_1;
+            p+=4;
+        }
+        while (p<=limit);
+
+        h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    }
+    else
+    {
+        h32  = seed + PRIME32_5;
+    }
+
+    h32 += (U32) len;
+
+    while (p+4<=bEnd)
+    {
+        h32 += XXH_get32bits(p) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h32 += (*p) * PRIME32_5;
+        h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+unsigned XXH32 (const void* input, size_t len, unsigned seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, input, len);
+    return XXH32_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+#  if !defined(XXH_USE_UNALIGNED_ACCESS)
+    if ((((size_t)input) & 3) == 0)   /* Input is 4-bytes aligned, leverage the speed benefit */
+    {
+        if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+            return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+        else
+            return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }
+#  endif
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U64 h64;
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL)
+    {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)32;
+    }
+#endif
+
+    if (len>=32)
+    {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = seed + PRIME64_1 + PRIME64_2;
+        U64 v2 = seed + PRIME64_2;
+        U64 v3 = seed + 0;
+        U64 v4 = seed - PRIME64_1;
+
+        do
+        {
+            v1 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v1 = XXH_rotl64(v1, 31);
+            v1 *= PRIME64_1;
+            v2 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v2 = XXH_rotl64(v2, 31);
+            v2 *= PRIME64_1;
+            v3 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v3 = XXH_rotl64(v3, 31);
+            v3 *= PRIME64_1;
+            v4 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v4 = XXH_rotl64(v4, 31);
+            v4 *= PRIME64_1;
+        }
+        while (p<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+
+        v1 *= PRIME64_2;
+        v1 = XXH_rotl64(v1, 31);
+        v1 *= PRIME64_1;
+        h64 ^= v1;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+
+        v2 *= PRIME64_2;
+        v2 = XXH_rotl64(v2, 31);
+        v2 *= PRIME64_1;
+        h64 ^= v2;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+
+        v3 *= PRIME64_2;
+        v3 = XXH_rotl64(v3, 31);
+        v3 *= PRIME64_1;
+        h64 ^= v3;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+
+        v4 *= PRIME64_2;
+        v4 = XXH_rotl64(v4, 31);
+        v4 *= PRIME64_1;
+        h64 ^= v4;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+    }
+    else
+    {
+        h64  = seed + PRIME64_5;
+    }
+
+    h64 += (U64) len;
+
+    while (p+8<=bEnd)
+    {
+        U64 k1 = XXH_get64bits(p);
+        k1 *= PRIME64_2;
+        k1 = XXH_rotl64(k1,31);
+        k1 *= PRIME64_1;
+        h64 ^= k1;
+        h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd)
+    {
+        h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h64 ^= (*p) * PRIME64_5;
+        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+
+unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, input, len);
+    return XXH64_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+#  if !defined(XXH_USE_UNALIGNED_ACCESS)
+    if ((((size_t)input) & 7)==0)   /* Input is aligned, let's leverage the speed advantage */
+    {
+        if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+            return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+        else
+            return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }
+#  endif
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/****************************************************
+*  Advanced Hash Functions
+****************************************************/
+
+/*** Allocation ***/
+typedef struct
+{
+    U64 total_len;
+    U32 seed;
+    U32 v1;
+    U32 v2;
+    U32 v3;
+    U32 v4;
+    U32 mem32[4];   /* defined as U32 for alignment */
+    U32 memsize;
+} XXH_istate32_t;
+
+typedef struct
+{
+    U64 total_len;
+    U64 seed;
+    U64 v1;
+    U64 v2;
+    U64 v3;
+    U64 v4;
+    U64 mem64[4];   /* defined as U64 for alignment */
+    U32 memsize;
+} XXH_istate64_t;
+
+
+XXH32_state_t* XXH32_createState(void)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t));   /* A compilation error here means XXH32_state_t is not large enough */
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH64_state_t* XXH64_createState(void)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t));   /* A compilation error here means XXH64_state_t is not large enough */
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+
+/*** Hash feed ***/
+
+XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed)
+{
+    XXH_istate32_t* state = (XXH_istate32_t*) state_in;
+    state->seed = seed;
+    state->v1 = seed + PRIME32_1 + PRIME32_2;
+    state->v2 = seed + PRIME32_2;
+    state->v3 = seed + 0;
+    state->v4 = seed - PRIME32_1;
+    state->total_len = 0;
+    state->memsize = 0;
+    return XXH_OK;
+}
+
+XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed)
+{
+    XXH_istate64_t* state = (XXH_istate64_t*) state_in;
+    state->seed = seed;
+    state->v1 = seed + PRIME64_1 + PRIME64_2;
+    state->v2 = seed + PRIME64_2;
+    state->v3 = seed + 0;
+    state->v4 = seed - PRIME64_1;
+    state->total_len = 0;
+    state->memsize = 0;
+    return XXH_OK;
+}
+
+
+FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
+{
+    XXH_istate32_t* state = (XXH_istate32_t *) state_in;
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 16)   /* fill in tmp buffer */
+    {
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize)   /* some data left from previous update */
+    {
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+        {
+            const U32* p32 = state->mem32;
+            state->v1 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v1 = XXH_rotl32(state->v1, 13);
+            state->v1 *= PRIME32_1;
+            p32++;
+            state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v2 = XXH_rotl32(state->v2, 13);
+            state->v2 *= PRIME32_1;
+            p32++;
+            state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v3 = XXH_rotl32(state->v3, 13);
+            state->v3 *= PRIME32_1;
+            p32++;
+            state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v4 = XXH_rotl32(state->v4, 13);
+            state->v4 *= PRIME32_1;
+            p32++;
+        }
+        p += 16-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p <= bEnd-16)
+    {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = state->v1;
+        U32 v2 = state->v2;
+        U32 v3 = state->v3;
+        U32 v4 = state->v4;
+
+        do
+        {
+            v1 += XXH_readLE32(p, endian) * PRIME32_2;
+            v1 = XXH_rotl32(v1, 13);
+            v1 *= PRIME32_1;
+            p+=4;
+            v2 += XXH_readLE32(p, endian) * PRIME32_2;
+            v2 = XXH_rotl32(v2, 13);
+            v2 *= PRIME32_1;
+            p+=4;
+            v3 += XXH_readLE32(p, endian) * PRIME32_2;
+            v3 = XXH_rotl32(v3, 13);
+            v3 *= PRIME32_1;
+            p+=4;
+            v4 += XXH_readLE32(p, endian) * PRIME32_2;
+            v4 = XXH_rotl32(v4, 13);
+            v4 *= PRIME32_1;
+            p+=4;
+        }
+        while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd)
+    {
+        XXH_memcpy(state->mem32, p, bEnd-p);
+        state->memsize = (int)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
+{
+    const XXH_istate32_t* state = (const XXH_istate32_t*) state_in;
+    const BYTE * p = (const BYTE*)state->mem32;
+    const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
+    U32 h32;
+
+    if (state->total_len >= 16)
+    {
+        h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+    }
+    else
+    {
+        h32  = state->seed + PRIME32_5;
+    }
+
+    h32 += (U32) state->total_len;
+
+    while (p+4<=bEnd)
+    {
+        h32 += XXH_readLE32(p, endian) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h32 += (*p) * PRIME32_5;
+        h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+U32 XXH32_digest (const XXH32_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
+{
+    XXH_istate64_t * state = (XXH_istate64_t *) state_in;
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 32)   /* fill in tmp buffer */
+    {
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize)   /* some data left from previous update */
+    {
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+        {
+            const U64* p64 = state->mem64;
+            state->v1 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v1 = XXH_rotl64(state->v1, 31);
+            state->v1 *= PRIME64_1;
+            p64++;
+            state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v2 = XXH_rotl64(state->v2, 31);
+            state->v2 *= PRIME64_1;
+            p64++;
+            state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v3 = XXH_rotl64(state->v3, 31);
+            state->v3 *= PRIME64_1;
+            p64++;
+            state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v4 = XXH_rotl64(state->v4, 31);
+            state->v4 *= PRIME64_1;
+            p64++;
+        }
+        p += 32-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p+32 <= bEnd)
+    {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        do
+        {
+            v1 += XXH_readLE64(p, endian) * PRIME64_2;
+            v1 = XXH_rotl64(v1, 31);
+            v1 *= PRIME64_1;
+            p+=8;
+            v2 += XXH_readLE64(p, endian) * PRIME64_2;
+            v2 = XXH_rotl64(v2, 31);
+            v2 *= PRIME64_1;
+            p+=8;
+            v3 += XXH_readLE64(p, endian) * PRIME64_2;
+            v3 = XXH_rotl64(v3, 31);
+            v3 *= PRIME64_1;
+            p+=8;
+            v4 += XXH_readLE64(p, endian) * PRIME64_2;
+            v4 = XXH_rotl64(v4, 31);
+            v4 *= PRIME64_1;
+            p+=8;
+        }
+        while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd)
+    {
+        XXH_memcpy(state->mem64, p, bEnd-p);
+        state->memsize = (int)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
+{
+    const XXH_istate64_t * state = (const XXH_istate64_t *) state_in;
+    const BYTE * p = (const BYTE*)state->mem64;
+    const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
+    U64 h64;
+
+    if (state->total_len >= 32)
+    {
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+
+        v1 *= PRIME64_2;
+        v1 = XXH_rotl64(v1, 31);
+        v1 *= PRIME64_1;
+        h64 ^= v1;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+
+        v2 *= PRIME64_2;
+        v2 = XXH_rotl64(v2, 31);
+        v2 *= PRIME64_1;
+        h64 ^= v2;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+
+        v3 *= PRIME64_2;
+        v3 = XXH_rotl64(v3, 31);
+        v3 *= PRIME64_1;
+        h64 ^= v3;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+
+        v4 *= PRIME64_2;
+        v4 = XXH_rotl64(v4, 31);
+        v4 *= PRIME64_1;
+        h64 ^= v4;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+    }
+    else
+    {
+        h64  = state->seed + PRIME64_5;
+    }
+
+    h64 += (U64) state->total_len;
+
+    while (p+8<=bEnd)
+    {
+        U64 k1 = XXH_readLE64(p, endian);
+        k1 *= PRIME64_2;
+        k1 = XXH_rotl64(k1,31);
+        k1 *= PRIME64_1;
+        h64 ^= k1;
+        h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd)
+    {
+        h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
+        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h64 ^= (*p) * PRIME64_5;
+        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+
+unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+

C/lz4/xxhash.h

@@ -0,0 +1,192 @@
+/*
+   xxHash - Extremely Fast Hash algorithm
+   Header File
+   Copyright (C) 2012-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name            Speed       Q.Score   Author
+xxHash          5.4 GB/s     10
+CrapWow         3.2 GB/s      2       Andrew
+MumurHash 3a    2.7 GB/s     10       Austin Appleby
+SpookyHash      2.0 GB/s     10       Bob Jenkins
+SBox            1.4 GB/s      9       Bret Mulvey
+Lookup3         1.2 GB/s      9       Bob Jenkins
+SuperFastHash   1.2 GB/s      1       Paul Hsieh
+CityHash64      1.05 GB/s    10       Pike & Alakuijala
+FNV             0.55 GB/s     5       Fowler, Noll, Vo
+CRC32           0.43 GB/s     9
+MD5-32          0.33 GB/s    10       Ronald L. Rivest
+SHA1-32         0.28 GB/s    10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bits version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bits applications only.
+Name     Speed on 64 bits    Speed on 32 bits
+XXH64       13.8 GB/s            1.9 GB/s
+XXH32        6.8 GB/s            6.0 GB/s
+*/
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/*****************************
+*  Definitions
+*****************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/*****************************
+*  Namespace Emulation
+*****************************/
+/* Motivations :
+
+If you need to include xxHash into your library,
+but wish to avoid xxHash symbols to be present on your library interface
+in an effort to avoid potential name collision if another library also includes xxHash,
+
+you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash
+with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values).
+
+Note that no change is required within the calling program :
+it can still call xxHash functions using their regular name.
+They will be automatically translated by this header.
+*/
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#endif
+
+
+/*****************************
+*  Simple Hash Functions
+*****************************/
+
+unsigned int       XXH32 (const void* input, size_t length, unsigned seed);
+unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*
+XXH32() :
+    Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
+    The memory between input & input+length must be valid (allocated and read-accessible).
+    "seed" can be used to alter the result predictably.
+    This function successfully passes all SMHasher tests.
+    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
+XXH64() :
+    Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+    Faster on 64-bits systems. Slower on 32-bits systems.
+*/
+
+
+
+/*****************************
+*  Advanced Hash Functions
+*****************************/
+typedef struct { long long ll[ 6]; } XXH32_state_t;
+typedef struct { long long ll[11]; } XXH64_state_t;
+
+/*
+These structures allow static allocation of XXH states.
+States must then be initialized using XXHnn_reset() before first use.
+
+If you prefer dynamic allocation, please refer to functions below.
+*/
+
+XXH32_state_t* XXH32_createState(void);
+XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+
+XXH64_state_t* XXH64_createState(void);
+XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+
+/*
+These functions create and release memory for XXH state.
+States must then be initialized using XXHnn_reset() before first use.
+*/
+
+
+XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned seed);
+XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+unsigned int  XXH32_digest (const XXH32_state_t* statePtr);
+
+XXH_errorcode      XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_errorcode      XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
+
+/*
+These functions calculate the xxHash of an input provided in multiple smaller packets,
+as opposed to an input provided as a single block.
+
+XXH state space must first be allocated, using either static or dynamic method provided above.
+
+Start a new hash by initializing state with a seed, using XXHnn_reset().
+
+Then, feed the hash state by calling XXHnn_update() as many times as necessary.
+Obviously, input must be valid, meaning allocated and read accessible.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+
+Finally, you can produce a hash anytime, by using XXHnn_digest().
+This function returns the final nn-bits hash.
+You can nonetheless continue feeding the hash state with more input,
+and therefore get some new hashes, by calling again XXHnn_digest().
+
+When you are done, don't forget to free XXH state space, using typically XXHnn_freeState().
+*/
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz5/lz5.h

@@ -0,0 +1,363 @@
+/*
+   LZ5 - Fast LZ compression algorithm
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ5 source repository : https://github.com/inikep/lz5
+   - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
+*/
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * lz5.h provides block compression functions, and gives full buffer control to programmer.
+ * If you need to generate inter-operable compressed data (respecting LZ5 frame specification),
+ * and can let the library handle its own memory, please use lz5frame.h instead.
+*/
+
+/**************************************
+*  Version
+**************************************/
+#define LZ5_VERSION          "v1.5.0"
+#define LZ5_VERSION_MAJOR    1    /* for breaking interface changes  */
+#define LZ5_VERSION_MINOR    5    /* for new (non-breaking) interface capabilities */
+#define LZ5_VERSION_RELEASE  0    /* for tweaks, bug-fixes, or development */
+#define LZ5_VERSION_NUMBER (LZ5_VERSION_MAJOR *100*100 + LZ5_VERSION_MINOR *100 + LZ5_VERSION_RELEASE)
+int LZ5_versionNumber (void);
+
+#define LZ5HC_MAX_CLEVEL     15
+
+
+/**************************************
+*  Tuning parameter
+**************************************/
+/*
+ * LZ5_MEMORY_USAGE :
+ * Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.)
+ * Increasing memory usage improves compression ratio
+ * Reduced memory usage can improve speed, due to cache effect
+ */
+#define LZ5_MEMORY_USAGE 20
+
+
+/**************************************
+*  Simple Functions
+**************************************/
+
+int LZ5_compress_default(const char* source, char* dest, int sourceSize, int maxDestSize);
+int LZ5_decompress_safe (const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+
+/*
+LZ5_compress_default() :
+    Compresses 'sourceSize' bytes from buffer 'source'
+    into already allocated 'dest' buffer of size 'maxDestSize'.
+    Compression is guaranteed to succeed if 'maxDestSize' >= LZ5_compressBound(sourceSize).
+    It also runs faster, so it's a recommended setting.
+    If the function cannot compress 'source' into a more limited 'dest' budget,
+    compression stops *immediately*, and the function result is zero.
+    As a consequence, 'dest' content is not valid.
+    This function never writes outside 'dest' buffer, nor read outside 'source' buffer.
+        sourceSize  : Max supported value is LZ5_MAX_INPUT_VALUE
+        maxDestSize : full or partial size of buffer 'dest' (which must be already allocated)
+        return : the number of bytes written into buffer 'dest' (necessarily <= maxOutputSize)
+              or 0 if compression fails
+
+LZ5_decompress_safe() :
+    compressedSize : is the precise full size of the compressed block.
+    maxDecompressedSize : is the size of destination buffer, which must be already allocated.
+    return : the number of bytes decompressed into destination buffer (necessarily <= maxDecompressedSize)
+             If destination buffer is not large enough, decoding will stop and output an error code (<0).
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             This function is protected against buffer overflow exploits, including malicious data packets.
+             It never writes outside output buffer, nor reads outside input buffer.
+*/
+
+
+/**************************************
+*  Advanced Functions
+**************************************/
+#define LZ5_MAX_INPUT_SIZE        0x7E000000   /* 2 113 929 216 bytes */
+#define LZ5_COMPRESSBOUND(isize)  ((unsigned)(isize) > (unsigned)LZ5_MAX_INPUT_SIZE ? 0 : (isize) + ((isize)/128) + 16)
+
+/*
+LZ5_compressBound() :
+    Provides the maximum size that LZ5 compression may output in a "worst case" scenario (input data not compressible)
+    This function is primarily useful for memory allocation purposes (destination buffer size).
+    Macro LZ5_COMPRESSBOUND() is also provided for compilation-time evaluation (stack memory allocation for example).
+    Note that LZ5_compress_default() compress faster when dest buffer size is >= LZ5_compressBound(srcSize)
+        inputSize  : max supported value is LZ5_MAX_INPUT_SIZE
+        return : maximum output size in a "worst case" scenario
+              or 0, if input size is too large ( > LZ5_MAX_INPUT_SIZE)
+*/
+int LZ5_compressBound(int inputSize);
+
+/*
+LZ5_compress_fast() :
+    Same as LZ5_compress_default(), but allows to select an "acceleration" factor.
+    The larger the acceleration value, the faster the algorithm, but also the lesser the compression.
+    It's a trade-off. It can be fine tuned, with each successive value providing roughly +~3% to speed.
+    An acceleration value of "1" is the same as regular LZ5_compress_default()
+    Values <= 0 will be replaced by ACCELERATION_DEFAULT (see lz5.c), which is 1.
+*/
+int LZ5_compress_fast (const char* source, char* dest, int sourceSize, int maxDestSize, int acceleration);
+
+
+/*
+LZ5_compress_fast_extState() :
+    Same compression function, just using an externally allocated memory space to store compression state.
+    Use LZ5_sizeofState() to know how much memory must be allocated,
+    and allocate it on 8-bytes boundaries (using malloc() typically).
+    Then, provide it as 'void* state' to compression function.
+*/
+int LZ5_sizeofState(void);
+int LZ5_compress_fast_extState (void* state, const char* source, char* dest, int inputSize, int maxDestSize, int acceleration);
+
+
+/*
+LZ5_compress_destSize() :
+    Reverse the logic, by compressing as much data as possible from 'source' buffer
+    into already allocated buffer 'dest' of size 'targetDestSize'.
+    This function either compresses the entire 'source' content into 'dest' if it's large enough,
+    or fill 'dest' buffer completely with as much data as possible from 'source'.
+        *sourceSizePtr : will be modified to indicate how many bytes where read from 'source' to fill 'dest'.
+                         New value is necessarily <= old value.
+        return : Nb bytes written into 'dest' (necessarily <= targetDestSize)
+              or 0 if compression fails
+*/
+int LZ5_compress_destSize (const char* source, char* dest, int* sourceSizePtr, int targetDestSize);
+
+
+/*
+LZ5_decompress_fast() :
+    originalSize : is the original and therefore uncompressed size
+    return : the number of bytes read from the source buffer (in other words, the compressed size)
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             Destination buffer must be already allocated. Its size must be a minimum of 'originalSize' bytes.
+    note : This function fully respect memory boundaries for properly formed compressed data.
+           It is a bit faster than LZ5_decompress_safe().
+           However, it does not provide any protection against intentionally modified data stream (malicious input).
+           Use this function in trusted environment only (data to decode comes from a trusted source).
+*/
+int LZ5_decompress_fast (const char* source, char* dest, int originalSize);
+
+/*
+LZ5_decompress_safe_partial() :
+    This function decompress a compressed block of size 'compressedSize' at position 'source'
+    into destination buffer 'dest' of size 'maxDecompressedSize'.
+    The function tries to stop decompressing operation as soon as 'targetOutputSize' has been reached,
+    reducing decompression time.
+    return : the number of bytes decoded in the destination buffer (necessarily <= maxDecompressedSize)
+       Note : this number can be < 'targetOutputSize' should the compressed block to decode be smaller.
+             Always control how many bytes were decoded.
+             If the source stream is detected malformed, the function will stop decoding and return a negative result.
+             This function never writes outside of output buffer, and never reads outside of input buffer. It is therefore protected against malicious data packets
+*/
+int LZ5_decompress_safe_partial (const char* source, char* dest, int compressedSize, int targetOutputSize, int maxDecompressedSize);
+
+
+/***********************************************
+*  Streaming Compression Functions
+***********************************************/
+#define LZ5_STREAMSIZE_U64 ((1 << (LZ5_MEMORY_USAGE-3)) + 4)
+#define LZ5_STREAMSIZE     (LZ5_STREAMSIZE_U64 * sizeof(long long))
+/*
+ * LZ5_stream_t
+ * information structure to track an LZ5 stream.
+ * important : init this structure content before first use !
+ * note : only allocated directly the structure if you are statically linking LZ5
+ *        If you are using liblz5 as a DLL, please use below construction methods instead.
+ */
+typedef struct { long long table[LZ5_STREAMSIZE_U64]; } LZ5_stream_t;
+
+/*
+ * LZ5_resetStream
+ * Use this function to init an allocated LZ5_stream_t structure
+ */
+void LZ5_resetStream (LZ5_stream_t* streamPtr);
+
+/*
+ * LZ5_createStream will allocate and initialize an LZ5_stream_t structure
+ * LZ5_freeStream releases its memory.
+ * In the context of a DLL (liblz5), please use these methods rather than the static struct.
+ * They are more future proof, in case of a change of LZ5_stream_t size.
+ */
+LZ5_stream_t* LZ5_createStream(void);
+int           LZ5_freeStream (LZ5_stream_t* streamPtr);
+
+/*
+ * LZ5_loadDict
+ * Use this function to load a static dictionary into LZ5_stream.
+ * Any previous data will be forgotten, only 'dictionary' will remain in memory.
+ * Loading a size of 0 is allowed.
+ * Return : dictionary size, in bytes (necessarily <= 64 KB)
+ */
+int LZ5_loadDict (LZ5_stream_t* streamPtr, const char* dictionary, int dictSize);
+
+/*
+ * LZ5_compress_fast_continue
+ * Compress buffer content 'src', using data from previously compressed blocks as dictionary to improve compression ratio.
+ * Important : Previous data blocks are assumed to still be present and unmodified !
+ * 'dst' buffer must be already allocated.
+ * If maxDstSize >= LZ5_compressBound(srcSize), compression is guaranteed to succeed, and runs faster.
+ * If not, and if compressed data cannot fit into 'dst' buffer size, compression stops, and function returns a zero.
+ */
+int LZ5_compress_fast_continue (LZ5_stream_t* streamPtr, const char* src, char* dst, int srcSize, int maxDstSize, int acceleration);
+
+/*
+ * LZ5_saveDict
+ * If previously compressed data block is not guaranteed to remain available at its memory location
+ * save it into a safer place (char* safeBuffer)
+ * Note : you don't need to call LZ5_loadDict() afterwards,
+ *        dictionary is immediately usable, you can therefore call LZ5_compress_fast_continue()
+ * Return : saved dictionary size in bytes (necessarily <= dictSize), or 0 if error
+ */
+int LZ5_saveDict (LZ5_stream_t* streamPtr, char* safeBuffer, int dictSize);
+
+
+/************************************************
+*  Streaming Decompression Functions
+************************************************/
+
+#define LZ5_STREAMDECODESIZE_U64  4
+#define LZ5_STREAMDECODESIZE     (LZ5_STREAMDECODESIZE_U64 * sizeof(unsigned long long))
+typedef struct { unsigned long long table[LZ5_STREAMDECODESIZE_U64]; } LZ5_streamDecode_t;
+/*
+ * LZ5_streamDecode_t
+ * information structure to track an LZ5 stream.
+ * init this structure content using LZ5_setStreamDecode or memset() before first use !
+ *
+ * In the context of a DLL (liblz5) please prefer usage of construction methods below.
+ * They are more future proof, in case of a change of LZ5_streamDecode_t size in the future.
+ * LZ5_createStreamDecode will allocate and initialize an LZ5_streamDecode_t structure
+ * LZ5_freeStreamDecode releases its memory.
+ */
+LZ5_streamDecode_t* LZ5_createStreamDecode(void);
+int                 LZ5_freeStreamDecode (LZ5_streamDecode_t* LZ5_stream);
+
+/*
+ * LZ5_setStreamDecode
+ * Use this function to instruct where to find the dictionary.
+ * Setting a size of 0 is allowed (same effect as reset).
+ * Return : 1 if OK, 0 if error
+ */
+int LZ5_setStreamDecode (LZ5_streamDecode_t* LZ5_streamDecode, const char* dictionary, int dictSize);
+
+/*
+*_continue() :
+    These decoding functions allow decompression of multiple blocks in "streaming" mode.
+    Previously decoded blocks *must* remain available at the memory position where they were decoded (up to 64 KB)
+    In the case of a ring buffers, decoding buffer must be either :
+    - Exactly same size as encoding buffer, with same update rule (block boundaries at same positions)
+      In which case, the decoding & encoding ring buffer can have any size, including very small ones ( < 64 KB).
+    - Larger than encoding buffer, by a minimum of maxBlockSize more bytes.
+      maxBlockSize is implementation dependent. It's the maximum size you intend to compress into a single block.
+      In which case, encoding and decoding buffers do not need to be synchronized,
+      and encoding ring buffer can have any size, including small ones ( < 64 KB).
+    - _At least_ 64 KB + 8 bytes + maxBlockSize.
+      In which case, encoding and decoding buffers do not need to be synchronized,
+      and encoding ring buffer can have any size, including larger than decoding buffer.
+    Whenever these conditions are not possible, save the last 64KB of decoded data into a safe buffer,
+    and indicate where it is saved using LZ5_setStreamDecode()
+*/
+int LZ5_decompress_safe_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int compressedSize, int maxDecompressedSize);
+int LZ5_decompress_fast_continue (LZ5_streamDecode_t* LZ5_streamDecode, const char* source, char* dest, int originalSize);
+
+
+/*
+Advanced decoding functions :
+*_usingDict() :
+    These decoding functions work the same as
+    a combination of LZ5_setStreamDecode() followed by LZ5_decompress_x_continue()
+    They are stand-alone. They don't need nor update an LZ5_streamDecode_t structure.
+*/
+int LZ5_decompress_safe_usingDict (const char* source, char* dest, int compressedSize, int maxDecompressedSize, const char* dictStart, int dictSize);
+int LZ5_decompress_fast_usingDict (const char* source, char* dest, int originalSize, const char* dictStart, int dictSize);
+
+
+/**************************************
+*  Obsolete Functions
+**************************************/
+/* Deprecate Warnings */
+/* Should these warnings messages be a problem,
+   it is generally possible to disable them,
+   with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual for example.
+   Otherwise, you can also define LZ5_DISABLE_DEPRECATE_WARNINGS */
+#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+#ifdef LZ5_DISABLE_DEPRECATE_WARNINGS
+#  define LZ5_DEPRECATED()   /* disable deprecation warnings */
+#else
+#  if (LZ5_GCC_VERSION >= 405) || defined(__clang__)
+#    define LZ5_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (LZ5_GCC_VERSION >= 301)
+#    define LZ5_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define LZ5_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement LZ5_DEPRECATED for this compiler")
+#    define LZ5_DEPRECATED(message)
+#  endif
+#endif /* LZ5_DISABLE_DEPRECATE_WARNINGS */
+
+/* Obsolete compression functions */
+/* These functions will generate warnings in a future release */
+int LZ5_compress               (const char* source, char* dest, int sourceSize);
+int LZ5_compress_limitedOutput (const char* source, char* dest, int sourceSize, int maxOutputSize);
+int LZ5_compress_withState               (void* state, const char* source, char* dest, int inputSize);
+int LZ5_compress_limitedOutput_withState (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ5_compress_continue                (LZ5_stream_t* LZ5_streamPtr, const char* source, char* dest, int inputSize);
+int LZ5_compress_limitedOutput_continue  (LZ5_stream_t* LZ5_streamPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+
+/* Obsolete decompression functions */
+/* These function names are completely deprecated and must no longer be used.
+   They are only provided in lz5.c for compatibility with older programs.
+    - LZ5_uncompress is the same as LZ5_decompress_fast
+    - LZ5_uncompress_unknownOutputSize is the same as LZ5_decompress_safe
+   These function prototypes are now disabled; uncomment them only if you really need them.
+   It is highly recommended to stop using these prototypes and migrate to maintained ones */
+/* int LZ5_uncompress (const char* source, char* dest, int outputSize); */
+/* int LZ5_uncompress_unknownOutputSize (const char* source, char* dest, int isize, int maxOutputSize); */
+
+/* Obsolete streaming functions; use new streaming interface whenever possible */
+LZ5_DEPRECATED("use LZ5_createStream() instead") void* LZ5_create (char* inputBuffer);
+LZ5_DEPRECATED("use LZ5_createStream() instead") int   LZ5_sizeofStreamState(void);
+LZ5_DEPRECATED("use LZ5_resetStream() instead")  int   LZ5_resetStreamState(void* state, char* inputBuffer);
+LZ5_DEPRECATED("use LZ5_saveDict() instead")     char* LZ5_slideInputBuffer (void* state);
+
+/* Obsolete streaming decoding functions */
+LZ5_DEPRECATED("use LZ5_decompress_safe_usingDict() instead") int LZ5_decompress_safe_withPrefix64k (const char* src, char* dst, int compressedSize, int maxDstSize);
+LZ5_DEPRECATED("use LZ5_decompress_fast_usingDict() instead") int LZ5_decompress_fast_withPrefix64k (const char* src, char* dst, int originalSize);
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz5/lz5common.h

@@ -0,0 +1,330 @@
+#ifndef LZ5COMMON_H
+#define LZ5COMMON_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/**************************************
+*  Tuning parameters
+**************************************/
+/*
+ * HEAPMODE :
+ * Select how default compression functions will allocate memory for their hash table,
+ * in memory stack (0:default, fastest), or in memory heap (1:requires malloc()).
+ */
+#ifdef _MSC_VER
+	#define HEAPMODE 1   /* Default stack size for VC++ is 1 MB and size of LZ5_stream_t exceeds that limit */ 
+#else
+	#define HEAPMODE 0
+#endif
+
+
+/*
+ * ACCELERATION_DEFAULT :
+ * Select "acceleration" for LZ5_compress_fast() when parameter value <= 0
+ */
+#define ACCELERATION_DEFAULT 1
+
+
+
+
+/**************************************
+*  Compiler Options
+**************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  define FORCE_INLINE static __forceinline
+#  include <intrin.h>
+#  pragma warning(disable : 4127)        /* disable: C4127: conditional expression is constant */
+#  pragma warning(disable : 4293)        /* disable: C4293: too large shift (32-bits) */
+#else
+#  if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)   /* C99 */
+#    if defined(__GNUC__) || defined(__clang__)
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif   /* __STDC_VERSION__ */
+#endif  /* _MSC_VER */
+
+#define LZ5_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+#if (LZ5_GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__)
+#  define expect(expr,value)    (__builtin_expect ((expr),(value)) )
+#else
+#  define expect(expr,value)    (expr)
+#endif
+
+#define likely(expr)     expect((expr) != 0, 1)
+#define unlikely(expr)   expect((expr) != 0, 0)
+
+
+
+/**************************************
+*  Memory routines
+**************************************/
+#include <stdlib.h>   /* malloc, calloc, free */
+#define ALLOCATOR(n,s) calloc(n,s)
+#define FREEMEM        free
+#include <string.h>   /* memset, memcpy */
+#define MEM_INIT       memset
+
+
+/**************************************
+*  Common Constants
+**************************************/
+#define MINMATCH 3 // should be 3 or 4
+
+#define WILDCOPYLENGTH 8
+#define LASTLITERALS 5
+#define MFLIMIT (WILDCOPYLENGTH+MINMATCH)
+static const int LZ5_minLength = (MFLIMIT+1);
+
+#define KB *(1 <<10)
+#define MB *(1 <<20)
+#define GB *(1U<<30)
+
+#define MAXD_LOG 22
+#define MAX_DISTANCE ((1 << MAXD_LOG) - 1)
+#define LZ5_DICT_SIZE (1 << MAXD_LOG)
+
+#define ML_BITS  3
+#define ML_MASK  ((1U<<ML_BITS)-1)
+#define RUN_BITS 3
+#define RUN_MASK ((1U<<RUN_BITS)-1)
+#define RUN_BITS2 2
+#define RUN_MASK2 ((1U<<RUN_BITS2)-1)
+#define ML_RUN_BITS (ML_BITS + RUN_BITS)
+#define ML_RUN_BITS2 (ML_BITS + RUN_BITS2)
+
+#define LZ5_SHORT_OFFSET_BITS 10
+#define LZ5_SHORT_OFFSET_DISTANCE (1<<LZ5_SHORT_OFFSET_BITS)
+#define LZ5_MID_OFFSET_BITS 16
+#define LZ5_MID_OFFSET_DISTANCE (1<<LZ5_MID_OFFSET_BITS)
+
+
+/**************************************
+*  Common Utils
+**************************************/
+#define LZ5_STATIC_ASSERT(c)    { enum { LZ5_static_assert = 1/(int)(!!(c)) }; }   /* use only *after* variable declarations */
+
+
+
+/****************************************************************
+*  Basic Types
+*****************************************************************/
+#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+# include <stdint.h>
+  typedef  uint8_t BYTE;
+  typedef uint16_t U16;
+  typedef  int16_t S16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+  typedef  int64_t S64;
+#else
+  typedef unsigned char       BYTE;
+  typedef unsigned short      U16;
+  typedef   signed short      S16;
+  typedef unsigned int        U32;
+  typedef   signed int        S32;
+  typedef unsigned long long  U64;
+  typedef   signed long long  S64;
+#endif
+
+
+/* *************************************
+*  HC Inline functions and Macros
+***************************************/
+#include "mem.h" // MEM_read
+#include "lz5.h" // LZ5HC_MAX_CLEVEL
+
+    
+static const U32 prime4bytes = 2654435761U;
+static const U64 prime5bytes = 889523592379ULL;
+
+#ifdef LZ5HC_INCLUDES
+static const U32 prime3bytes = 506832829U;
+static const U64 prime6bytes = 227718039650203ULL;
+static const U64 prime7bytes = 58295818150454627ULL;
+
+static U32 LZ5HC_hash3(U32 u, U32 h) { return (u * prime3bytes) << (32-24) >> (32-h) ; }
+static size_t LZ5HC_hash3Ptr(const void* ptr, U32 h) { return LZ5HC_hash3(MEM_read32(ptr), h); }
+
+static U32 LZ5HC_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; }
+static size_t LZ5HC_hash4Ptr(const void* ptr, U32 h) { return LZ5HC_hash4(MEM_read32(ptr), h); }
+
+static size_t LZ5HC_hash5(U64 u, U32 h) { return (size_t)((u * prime5bytes) << (64-40) >> (64-h)) ; }
+static size_t LZ5HC_hash5Ptr(const void* p, U32 h) { return LZ5HC_hash5(MEM_read64(p), h); }
+
+static size_t LZ5HC_hash6(U64 u, U32 h) { return (size_t)((u * prime6bytes) << (64-48) >> (64-h)) ; }
+static size_t LZ5HC_hash6Ptr(const void* p, U32 h) { return LZ5HC_hash6(MEM_read64(p), h); }
+
+static size_t LZ5HC_hash7(U64 u, U32 h) { return (size_t)((u * prime7bytes) << (64-56) >> (64-h)) ; }
+static size_t LZ5HC_hash7Ptr(const void* p, U32 h) { return LZ5HC_hash7(MEM_read64(p), h); }
+
+static size_t LZ5HC_hashPtr(const void* p, U32 hBits, U32 mls)
+{
+    switch(mls)
+    {
+    default:
+    case 4: return LZ5HC_hash4Ptr(p, hBits);
+    case 5: return LZ5HC_hash5Ptr(p, hBits);
+    case 6: return LZ5HC_hash6Ptr(p, hBits);
+    case 7: return LZ5HC_hash7Ptr(p, hBits);
+    }
+}
+
+
+/**************************************
+*  HC Local Macros
+**************************************/
+#define LZ5HC_DEBUG(fmt, ...) //printf(fmt, __VA_ARGS__)
+#define LZ5_LOG_PARSER(fmt, ...) //printf(fmt, __VA_ARGS__)
+#define LZ5_LOG_PRICE(fmt, ...) //printf(fmt, __VA_ARGS__)
+#define LZ5_LOG_ENCODE(fmt, ...) //printf(fmt, __VA_ARGS__)
+
+#define MAX(a,b) ((a)>(b))?(a):(b)
+#define LZ5_OPT_NUM   (1<<12)
+
+#define LZ5_SHORT_LITERALS          ((1<<RUN_BITS2)-1)
+#define LZ5_LITERALS                ((1<<RUN_BITS)-1)
+
+#define LZ5_SHORT_LITLEN_COST(len)  (len<LZ5_SHORT_LITERALS ? 0 : (len-LZ5_SHORT_LITERALS < 255 ? 1 : (len-LZ5_SHORT_LITERALS-255 < (1<<7) ? 2 : 3)))
+#define LZ5_LEN_COST(len)           (len<LZ5_LITERALS ? 0 : (len-LZ5_LITERALS < 255 ? 1 : (len-LZ5_LITERALS-255 < (1<<7) ? 2 : 3)))
+
+static size_t LZ5_LIT_COST(size_t len, size_t offset){ return (len)+(((offset > LZ5_MID_OFFSET_DISTANCE) || (offset<LZ5_SHORT_OFFSET_DISTANCE)) ? LZ5_SHORT_LITLEN_COST(len) : LZ5_LEN_COST(len)); }
+static size_t LZ5_MATCH_COST(size_t mlen, size_t offset) { return LZ5_LEN_COST(mlen) + ((offset == 0) ? 1 : (offset<LZ5_SHORT_OFFSET_DISTANCE ? 2 : (offset<LZ5_MID_OFFSET_DISTANCE ? 3 : 4))); }
+
+#define LZ5_CODEWORD_COST(litlen,offset,mlen)   (LZ5_MATCH_COST(mlen,offset) + LZ5_LIT_COST(litlen,offset))
+#define LZ5_LIT_ONLY_COST(len)                  ((len)+(LZ5_LEN_COST(len))+1)
+
+#define LZ5_NORMAL_MATCH_COST(mlen,offset)  (LZ5_MATCH_COST(mlen,offset))
+#define LZ5_NORMAL_LIT_COST(len)            (len)
+
+
+
+FORCE_INLINE size_t LZ5HC_get_price(size_t litlen, size_t offset, size_t mlen)
+{
+	return LZ5_CODEWORD_COST(litlen, offset, mlen);
+}
+
+FORCE_INLINE size_t LZ5HC_better_price(size_t best_off, size_t best_common, size_t off, size_t common, size_t last_off)
+{
+  return LZ5_NORMAL_MATCH_COST(common - MINMATCH, (off == last_off) ? 0 : off) < LZ5_NORMAL_MATCH_COST(best_common - MINMATCH, (best_off == last_off) ? 0 : best_off) + (LZ5_NORMAL_LIT_COST(common - best_common) );
+}
+
+
+FORCE_INLINE size_t LZ5HC_more_profitable(size_t best_off, size_t best_common, size_t off, size_t common, size_t literals, size_t last_off)
+{
+	size_t sum;
+	
+	if (literals > 0)
+		sum = MAX(common + literals, best_common);
+	else
+		sum = MAX(common, best_common - literals);
+	
+//	return LZ5_CODEWORD_COST(sum - common, (off == last_off) ? 0 : (off), common - MINMATCH) <= LZ5_CODEWORD_COST(sum - best_common, (best_off == last_off) ? 0 : (best_off), best_common - MINMATCH);
+	return LZ5_NORMAL_MATCH_COST(common - MINMATCH, (off == last_off) ? 0 : off) + LZ5_NORMAL_LIT_COST(sum - common) <= LZ5_NORMAL_MATCH_COST(best_common - MINMATCH, (best_off == last_off) ? 0 : (best_off)) + LZ5_NORMAL_LIT_COST(sum - best_common);
+}
+
+#endif // LZ5HC_INCLUDES
+
+
+
+/* *************************************
+*  HC Types
+***************************************/
+/** from faster to stronger */
+typedef enum { LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price, LZ5HC_optimal_price, LZ5HC_optimal_price_bt } LZ5HC_strategy;
+
+typedef struct
+{
+    U32 windowLog;     /* largest match distance : impact decompression buffer size */
+    U32 contentLog;    /* full search segment : larger == more compression, slower, more memory (useless for fast) */
+    U32 hashLog;       /* dispatch table : larger == more memory, faster*/
+    U32 hashLog3;      /* dispatch table : larger == more memory, faster*/
+    U32 searchNum;     /* nb of searches : larger == more compression, slower*/
+    U32 searchLength;  /* size of matches : larger == faster decompression */
+    U32 sufficientLength;  /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */
+    U32 fullSearch;    /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */
+    LZ5HC_strategy strategy;
+} LZ5HC_parameters;
+
+
+struct LZ5HC_Data_s
+{
+    U32*   hashTable;
+    U32*   hashTable3;
+    U32*   chainTable;
+    const BYTE* end;        /* next block here to continue on current prefix */
+    const BYTE* base;       /* All index relative to this position */
+    const BYTE* dictBase;   /* alternate base for extDict */
+    const BYTE* inputBuffer;      /* for debugging */
+    const BYTE* outputBuffer;     /* for debugging */
+    U32   dictLimit;        /* below that point, need extDict */
+    U32   lowLimit;         /* below that point, no more dict */
+    U32   nextToUpdate;     /* index from which to continue dictionary update */
+    U32   compressionLevel;
+    U32   last_off;
+    LZ5HC_parameters params;
+};
+
+typedef struct
+{
+	int off;
+	int len;
+	int back;
+} LZ5HC_match_t;
+
+typedef struct
+{
+	int price;
+	int off;
+	int mlen;
+	int litlen;
+   	int rep;
+} LZ5HC_optimal_t;
+
+
+
+/* *************************************
+*  HC Pre-defined compression levels
+***************************************/
+
+static const int g_maxCompressionLevel = LZ5HC_MAX_CLEVEL;
+static const int LZ5HC_compressionLevel_default = 6;
+
+static const LZ5HC_parameters LZ5HC_defaultParameters[LZ5HC_MAX_CLEVEL+1] =
+{
+    /* windLog, contentLog,  H, H3,  Snum, SL, SuffL, FS, Strategy */
+    {        0,          0,  0,  0,     0,  0,     0,  0, LZ5HC_fast             }, // level 0 - never used
+    { MAXD_LOG,   MAXD_LOG, 13,  0,     4,  6,     0,  0, LZ5HC_fast             }, // level 1
+    { MAXD_LOG,   MAXD_LOG, 13,  0,     2,  6,     0,  0, LZ5HC_fast             }, // level 2
+    { MAXD_LOG,   MAXD_LOG, 13,  0,     1,  5,     0,  0, LZ5HC_fast             }, // level 3
+    { MAXD_LOG,   MAXD_LOG, 14, 13,     1,  4,     0,  0, LZ5HC_price_fast       }, // level 4
+    { MAXD_LOG,   MAXD_LOG, 17, 13,     1,  4,     0,  0, LZ5HC_price_fast       }, // level 5
+    { MAXD_LOG,   MAXD_LOG, 15, 13,     1,  4,     0,  0, LZ5HC_lowest_price     }, // level 6
+    { MAXD_LOG,   MAXD_LOG, 17, 13,     1,  4,     0,  0, LZ5HC_lowest_price     }, // level 7
+    { MAXD_LOG,   MAXD_LOG, 19, 16,     1,  4,     0,  0, LZ5HC_lowest_price     }, // level 8
+    { MAXD_LOG,   MAXD_LOG, 23, 16,     3,  4,     0,  0, LZ5HC_lowest_price     }, // level 9
+    { MAXD_LOG,   MAXD_LOG, 23, 16,     8,  4,     0,  0, LZ5HC_lowest_price     }, // level 10
+    { MAXD_LOG,   MAXD_LOG, 23, 16,     8,  4,    12,  0, LZ5HC_optimal_price    }, // level 11
+    { MAXD_LOG,   MAXD_LOG, 23, 16,     8,  4,    64,  0, LZ5HC_optimal_price    }, // level 12
+    { MAXD_LOG, MAXD_LOG+1, 23, 16,     8,  4,    64,  1, LZ5HC_optimal_price_bt }, // level 13
+    { MAXD_LOG, MAXD_LOG+1, 23, 16,   128,  4,    64,  1, LZ5HC_optimal_price_bt }, // level 14
+    { MAXD_LOG, MAXD_LOG+1, 28, 24, 1<<10,  4, 1<<10,  1, LZ5HC_optimal_price_bt }, // level 15
+//  {       10,         10, 10,  0,     0,  4,     0,  0, LZ5HC_fast          }, // min values
+//  {       24,         24, 28, 24, 1<<24,  7, 1<<24,  2, LZ5HC_optimal_price }, // max values
+};
+
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* LZ5COMMON_H */

C/lz5/lz5frame.h

@@ -0,0 +1,306 @@
+/*
+   LZ5 auto-framing library
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ5 source repository : https://github.com/inikep/lz5
+   - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
+*/
+
+/* LZ5F is a stand-alone API to create LZ5-compressed frames
+ * conformant with specification v1.5.1.
+ * All related operations, including memory management, are handled internally by the library.
+ * You don't need lz5.h when using lz5frame.h.
+ * */
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/**************************************
+*  Includes
+**************************************/
+#include <stddef.h>   /* size_t */
+
+
+/**************************************
+*  Error management
+**************************************/
+typedef size_t LZ5F_errorCode_t;
+
+unsigned    LZ5F_isError(LZ5F_errorCode_t code);
+const char* LZ5F_getErrorName(LZ5F_errorCode_t code);   /* return error code string; useful for debugging */
+
+
+/**************************************
+*  Frame compression types
+**************************************/
+//#define LZ5F_DISABLE_OBSOLETE_ENUMS
+#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS
+#  define LZ5F_OBSOLETE_ENUM(x) ,x
+#else
+#  define LZ5F_OBSOLETE_ENUM(x)
+#endif
+
+typedef enum {
+    LZ5F_default=0,
+    LZ5F_max64KB=1,
+    LZ5F_max256KB=2,
+    LZ5F_max1MB=3,
+    LZ5F_max4MB=4,
+    LZ5F_max16MB=5,
+    LZ5F_max64MB=6,
+    LZ5F_max256MB=7
+    LZ5F_OBSOLETE_ENUM(max64KB = LZ5F_max64KB)
+    LZ5F_OBSOLETE_ENUM(max256KB = LZ5F_max256KB)
+    LZ5F_OBSOLETE_ENUM(max1MB = LZ5F_max1MB)
+    LZ5F_OBSOLETE_ENUM(max4MB = LZ5F_max4MB)
+} LZ5F_blockSizeID_t;
+
+typedef enum {
+    LZ5F_blockLinked=0,
+    LZ5F_blockIndependent
+    LZ5F_OBSOLETE_ENUM(blockLinked = LZ5F_blockLinked)
+    LZ5F_OBSOLETE_ENUM(blockIndependent = LZ5F_blockIndependent)
+} LZ5F_blockMode_t;
+
+typedef enum {
+    LZ5F_noContentChecksum=0,
+    LZ5F_contentChecksumEnabled
+    LZ5F_OBSOLETE_ENUM(noContentChecksum = LZ5F_noContentChecksum)
+    LZ5F_OBSOLETE_ENUM(contentChecksumEnabled = LZ5F_contentChecksumEnabled)
+} LZ5F_contentChecksum_t;
+
+typedef enum {
+    LZ5F_frame=0,
+    LZ5F_skippableFrame
+    LZ5F_OBSOLETE_ENUM(skippableFrame = LZ5F_skippableFrame)
+} LZ5F_frameType_t;
+
+#ifndef LZ5F_DISABLE_OBSOLETE_ENUMS
+typedef LZ5F_blockSizeID_t blockSizeID_t;
+typedef LZ5F_blockMode_t blockMode_t;
+typedef LZ5F_frameType_t frameType_t;
+typedef LZ5F_contentChecksum_t contentChecksum_t;
+#endif
+
+typedef struct {
+  LZ5F_blockSizeID_t     blockSizeID;           /* max64KB, max256KB, max1MB, max4MB ; 0 == default */
+  LZ5F_blockMode_t       blockMode;             /* blockLinked, blockIndependent ; 0 == default */
+  LZ5F_contentChecksum_t contentChecksumFlag;   /* noContentChecksum, contentChecksumEnabled ; 0 == default  */
+  LZ5F_frameType_t       frameType;             /* LZ5F_frame, skippableFrame ; 0 == default */
+  unsigned long long     contentSize;           /* Size of uncompressed (original) content ; 0 == unknown */
+  unsigned               reserved[2];           /* must be zero for forward compatibility */
+} LZ5F_frameInfo_t;
+
+typedef struct {
+  LZ5F_frameInfo_t frameInfo;
+  int      compressionLevel;       /* 0 == default (fast mode); values above 16 count as 16; values below 0 count as 0 */
+  unsigned autoFlush;              /* 1 == always flush (reduce need for tmp buffer) */
+  unsigned reserved[4];            /* must be zero for forward compatibility */
+} LZ5F_preferences_t;
+
+
+/***********************************
+*  Simple compression function
+***********************************/
+size_t LZ5F_compressFrameBound(size_t srcSize, const LZ5F_preferences_t* preferencesPtr);
+
+size_t LZ5F_compressFrame(void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_preferences_t* preferencesPtr);
+/* LZ5F_compressFrame()
+ * Compress an entire srcBuffer into a valid LZ5 frame, as defined by specification v1.5.1
+ * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
+ * You can get the minimum value of dstMaxSize by using LZ5F_compressFrameBound()
+ * If this condition is not respected, LZ5F_compressFrame() will fail (result is an errorCode)
+ * The LZ5F_preferences_t structure is optional : you can provide NULL as argument. All preferences will be set to default.
+ * The result of the function is the number of bytes written into dstBuffer.
+ * The function outputs an error code if it fails (can be tested using LZ5F_isError())
+ */
+
+
+
+/**********************************
+*  Advanced compression functions
+**********************************/
+typedef struct LZ5F_cctx_s* LZ5F_compressionContext_t;   /* must be aligned on 8-bytes */
+
+typedef struct {
+  unsigned stableSrc;    /* 1 == src content will remain available on future calls to LZ5F_compress(); avoid saving src content within tmp buffer as future dictionary */
+  unsigned reserved[3];
+} LZ5F_compressOptions_t;
+
+/* Resource Management */
+
+#define LZ5F_VERSION 100
+LZ5F_errorCode_t LZ5F_createCompressionContext(LZ5F_compressionContext_t* cctxPtr, unsigned version);
+LZ5F_errorCode_t LZ5F_freeCompressionContext(LZ5F_compressionContext_t cctx);
+/* LZ5F_createCompressionContext() :
+ * The first thing to do is to create a compressionContext object, which will be used in all compression operations.
+ * This is achieved using LZ5F_createCompressionContext(), which takes as argument a version and an LZ5F_preferences_t structure.
+ * The version provided MUST be LZ5F_VERSION. It is intended to track potential version differences between different binaries.
+ * The function will provide a pointer to a fully allocated LZ5F_compressionContext_t object.
+ * If the result LZ5F_errorCode_t is not zero, there was an error during context creation.
+ * Object can release its memory using LZ5F_freeCompressionContext();
+ */
+
+
+/* Compression */
+
+size_t LZ5F_compressBegin(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_preferences_t* prefsPtr);
+/* LZ5F_compressBegin() :
+ * will write the frame header into dstBuffer.
+ * dstBuffer must be large enough to accommodate a header (dstMaxSize). Maximum header size is 15 bytes.
+ * The LZ5F_preferences_t structure is optional : you can provide NULL as argument, all preferences will then be set to default.
+ * The result of the function is the number of bytes written into dstBuffer for the header
+ * or an error code (can be tested using LZ5F_isError())
+ */
+
+size_t LZ5F_compressBound(size_t srcSize, const LZ5F_preferences_t* prefsPtr);
+/* LZ5F_compressBound() :
+ * Provides the minimum size of Dst buffer given srcSize to handle worst case situations.
+ * Different preferences can produce different results.
+ * prefsPtr is optional : you can provide NULL as argument, all preferences will then be set to cover worst case.
+ * This function includes frame termination cost (4 bytes, or 8 if frame checksum is enabled)
+ */
+
+size_t LZ5F_compressUpdate(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const void* srcBuffer, size_t srcSize, const LZ5F_compressOptions_t* cOptPtr);
+/* LZ5F_compressUpdate()
+ * LZ5F_compressUpdate() can be called repetitively to compress as much data as necessary.
+ * The most important rule is that dstBuffer MUST be large enough (dstMaxSize) to ensure compression completion even in worst case.
+ * You can get the minimum value of dstMaxSize by using LZ5F_compressBound().
+ * If this condition is not respected, LZ5F_compress() will fail (result is an errorCode).
+ * LZ5F_compressUpdate() doesn't guarantee error recovery, so you have to reset compression context when an error occurs.
+ * The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * The result of the function is the number of bytes written into dstBuffer : it can be zero, meaning input data was just buffered.
+ * The function outputs an error code if it fails (can be tested using LZ5F_isError())
+ */
+
+size_t LZ5F_flush(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr);
+/* LZ5F_flush()
+ * Should you need to generate compressed data immediately, without waiting for the current block to be filled,
+ * you can call LZ5_flush(), which will immediately compress any remaining data buffered within cctx.
+ * Note that dstMaxSize must be large enough to ensure the operation will be successful.
+ * LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * The result of the function is the number of bytes written into dstBuffer
+ * (it can be zero, this means there was no data left within cctx)
+ * The function outputs an error code if it fails (can be tested using LZ5F_isError())
+ */
+
+size_t LZ5F_compressEnd(LZ5F_compressionContext_t cctx, void* dstBuffer, size_t dstMaxSize, const LZ5F_compressOptions_t* cOptPtr);
+/* LZ5F_compressEnd()
+ * When you want to properly finish the compressed frame, just call LZ5F_compressEnd().
+ * It will flush whatever data remained within compressionContext (like LZ5_flush())
+ * but also properly finalize the frame, with an endMark and a checksum.
+ * The result of the function is the number of bytes written into dstBuffer (necessarily >= 4 (endMark), or 8 if optional frame checksum is enabled)
+ * The function outputs an error code if it fails (can be tested using LZ5F_isError())
+ * The LZ5F_compressOptions_t structure is optional : you can provide NULL as argument.
+ * A successful call to LZ5F_compressEnd() makes cctx available again for next compression task.
+ */
+
+
+/***********************************
+*  Decompression functions
+***********************************/
+
+typedef struct LZ5F_dctx_s* LZ5F_decompressionContext_t;   /* must be aligned on 8-bytes */
+
+typedef struct {
+  unsigned stableDst;       /* guarantee that decompressed data will still be there on next function calls (avoid storage into tmp buffers) */
+  unsigned reserved[3];
+} LZ5F_decompressOptions_t;
+
+
+/* Resource management */
+
+LZ5F_errorCode_t LZ5F_createDecompressionContext(LZ5F_decompressionContext_t* dctxPtr, unsigned version);
+LZ5F_errorCode_t LZ5F_freeDecompressionContext(LZ5F_decompressionContext_t dctx);
+/* LZ5F_createDecompressionContext() :
+ * The first thing to do is to create an LZ5F_decompressionContext_t object, which will be used in all decompression operations.
+ * This is achieved using LZ5F_createDecompressionContext().
+ * The version provided MUST be LZ5F_VERSION. It is intended to track potential breaking differences between different versions.
+ * The function will provide a pointer to a fully allocated and initialized LZ5F_decompressionContext_t object.
+ * The result is an errorCode, which can be tested using LZ5F_isError().
+ * dctx memory can be released using LZ5F_freeDecompressionContext();
+ * The result of LZ5F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released.
+ * That is, it should be == 0 if decompression has been completed fully and correctly.
+ */
+
+
+/* Decompression */
+
+size_t LZ5F_getFrameInfo(LZ5F_decompressionContext_t dctx,
+                         LZ5F_frameInfo_t* frameInfoPtr,
+                         const void* srcBuffer, size_t* srcSizePtr);
+/* LZ5F_getFrameInfo()
+ * This function decodes frame header information (such as max blockSize, frame checksum, etc.).
+ * Its usage is optional. The objective is to extract frame header information, typically for allocation purposes.
+ * A header size is variable and can be from 7 to 15 bytes. It's also possible to input more bytes than that. 
+ * The number of bytes read from srcBuffer will be updated within *srcSizePtr (necessarily <= original value).
+ * (note that LZ5F_getFrameInfo() can also be used anytime *after* starting decompression, in this case 0 input byte is enough)
+ * Frame header info is *copied into* an already allocated LZ5F_frameInfo_t structure.
+ * The function result is an hint about how many srcSize bytes LZ5F_decompress() expects for next call,
+ *                        or an error code which can be tested using LZ5F_isError()
+ *                        (typically, when there is not enough src bytes to fully decode the frame header)
+ * Decompression is expected to resume from where it stopped (srcBuffer + *srcSizePtr)
+ */
+
+size_t LZ5F_decompress(LZ5F_decompressionContext_t dctx,
+                       void* dstBuffer, size_t* dstSizePtr,
+                       const void* srcBuffer, size_t* srcSizePtr,
+                       const LZ5F_decompressOptions_t* dOptPtr);
+/* LZ5F_decompress()
+ * Call this function repetitively to regenerate data compressed within srcBuffer.
+ * The function will attempt to decode *srcSizePtr bytes from srcBuffer, into dstBuffer of maximum size *dstSizePtr.
+ *
+ * The number of bytes regenerated into dstBuffer will be provided within *dstSizePtr (necessarily <= original value).
+ *
+ * The number of bytes read from srcBuffer will be provided within *srcSizePtr (necessarily <= original value).
+ * If number of bytes read is < number of bytes provided, then decompression operation is not completed.
+ * It typically happens when dstBuffer is not large enough to contain all decoded data.
+ * LZ5F_decompress() must be called again, starting from where it stopped (srcBuffer + *srcSizePtr)
+ * The function will check this condition, and refuse to continue if it is not respected.
+ *
+ * dstBuffer is supposed to be flushed between each call to the function, since its content will be overwritten.
+ * dst arguments can be changed at will with each consecutive call to the function.
+ *
+ * The function result is an hint of how many srcSize bytes LZ5F_decompress() expects for next call.
+ * Schematically, it's the size of the current (or remaining) compressed block + header of next block.
+ * Respecting the hint provides some boost to performance, since it does skip intermediate buffers.
+ * This is just a hint, you can always provide any srcSize you want.
+ * When a frame is fully decoded, the function result will be 0 (no more data expected).
+ * If decompression failed, function result is an error code, which can be tested using LZ5F_isError().
+ *
+ * After a frame is fully decoded, dctx can be used again to decompress another frame.
+ */
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz5/lz5frame_static.h

@@ -0,0 +1,81 @@
+/*
+   LZ5 auto-framing library
+   Header File for static linking only
+   Copyright (C) 2011-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ5 source repository : https://github.com/inikep/lz5
+   - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
+*/
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/* lz5frame_static.h should be used solely in the context of static linking.
+ * It contains definitions which may still change overtime.
+ * Never use it in the context of DLL linking.
+ * */
+
+
+/**************************************
+*  Includes
+**************************************/
+#include "lz5frame.h"
+
+
+/**************************************
+ * Error management
+ * ************************************/
+#define LZ5F_LIST_ERRORS(ITEM) \
+        ITEM(OK_NoError) ITEM(ERROR_GENERIC) \
+        ITEM(ERROR_maxBlockSize_invalid) ITEM(ERROR_blockMode_invalid) ITEM(ERROR_contentChecksumFlag_invalid) \
+        ITEM(ERROR_compressionLevel_invalid) \
+        ITEM(ERROR_headerVersion_wrong) ITEM(ERROR_blockChecksum_unsupported) ITEM(ERROR_reservedFlag_set) \
+        ITEM(ERROR_allocation_failed) \
+        ITEM(ERROR_srcSize_tooLarge) ITEM(ERROR_dstMaxSize_tooSmall) \
+        ITEM(ERROR_frameHeader_incomplete) ITEM(ERROR_frameType_unknown) ITEM(ERROR_frameSize_wrong) \
+        ITEM(ERROR_srcPtr_wrong) \
+        ITEM(ERROR_decompressionFailed) \
+        ITEM(ERROR_headerChecksum_invalid) ITEM(ERROR_contentChecksum_invalid) \
+        ITEM(ERROR_maxCode)
+
+//#define LZ5F_DISABLE_OLD_ENUMS
+#ifndef LZ5F_DISABLE_OLD_ENUMS
+#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM, ENUM = LZ5F_##ENUM,
+#else
+#define LZ5F_GENERATE_ENUM(ENUM) LZ5F_##ENUM,
+#endif
+typedef enum { LZ5F_LIST_ERRORS(LZ5F_GENERATE_ENUM) } LZ5F_errorCodes;  /* enum is exposed, to handle specific errors; compare function result to -enum value */
+
+
+#if defined (__cplusplus)
+}
+#endif

C/lz5/lz5hc.h

@@ -0,0 +1,178 @@
+/*
+   LZ5 HC - High Compression Mode of LZ5
+   Header File
+   Copyright (C) 2011-2015, Yann Collet.
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - LZ5 source repository : https://github.com/inikep/lz5
+   - LZ5 public forum : https://groups.google.com/forum/#!forum/lz5c
+*/
+#pragma once
+
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*****************************
+*  Includes
+*****************************/
+#include <stddef.h>   /* size_t */
+
+
+/**************************************
+*  Block Compression
+**************************************/
+int LZ5_compress_HC (const char* src, char* dst, int srcSize, int maxDstSize, int compressionLevel);
+/*
+LZ5_compress_HC :
+    Destination buffer 'dst' must be already allocated.
+    Compression completion is guaranteed if 'dst' buffer is sized to handle worst circumstances (data not compressible)
+    Worst size evaluation is provided by function LZ5_compressBound() (see "lz5.h")
+      srcSize  : Max supported value is LZ5_MAX_INPUT_SIZE (see "lz5.h")
+      compressionLevel : Recommended values are between 4 and 9, although any value between 0 and LZ5HC_MAX_CLEVEL (equal to 15) will work.
+                         0 means "use default value" (see lz5hc.c).
+                         Values >LZ5HC_MAX_CLEVEL behave the same as LZ5HC_MAX_CLEVEL.
+      return : the number of bytes written into buffer 'dst'
+            or 0 if compression fails.
+*/
+
+
+/* Note :
+   Decompression functions are provided within LZ5 source code (see "lz5.h") (BSD license)
+*/
+
+typedef struct LZ5HC_Data_s LZ5HC_Data_Structure;
+
+int LZ5_alloc_mem_HC(LZ5HC_Data_Structure* statePtr, int compressionLevel);
+void LZ5_free_mem_HC(LZ5HC_Data_Structure* statePtr);
+
+int LZ5_sizeofStateHC(void);
+int LZ5_compress_HC_extStateHC(void* state, const char* src, char* dst, int srcSize, int maxDstSize);
+/*
+LZ5_compress_HC_extStateHC() :
+   Use this function if you prefer to manually allocate memory for compression tables.
+   To know how much memory must be allocated for the compression tables, use :
+      int LZ5_sizeofStateHC();
+
+   Allocated memory must be aligned on 8-bytes boundaries (which a normal malloc() will do properly).
+
+   The allocated memory can then be provided to the compression functions using 'void* state' parameter.
+   LZ5_compress_HC_extStateHC() is equivalent to previously described function.
+   It just uses externally allocated memory for stateHC.
+*/
+
+
+/**************************************
+*  Streaming Compression
+**************************************/
+#define LZ5_STREAMHCSIZE        262192
+#define LZ5_STREAMHCSIZE_SIZET (LZ5_STREAMHCSIZE / sizeof(size_t))
+typedef struct { size_t table[LZ5_STREAMHCSIZE_SIZET]; } LZ5_streamHC_t;
+/*
+  LZ5_streamHC_t
+  This structure allows static allocation of LZ5 HC streaming state.
+  State must then be initialized using LZ5_resetStreamHC() before first use.
+
+  Static allocation should only be used in combination with static linking.
+  If you want to use LZ5 as a DLL, please use construction functions below, which are future-proof.
+*/
+
+
+LZ5_streamHC_t* LZ5_createStreamHC(int compressionLevel);
+int             LZ5_freeStreamHC (LZ5_streamHC_t* streamHCPtr);
+/*
+  These functions create and release memory for LZ5 HC streaming state.
+  Newly created states are already initialized.
+  Existing state space can be re-used anytime using LZ5_resetStreamHC().
+  If you use LZ5 as a DLL, use these functions instead of static structure allocation,
+  to avoid size mismatch between different versions.
+*/
+
+void LZ5_resetStreamHC (LZ5_streamHC_t* streamHCPtr);
+int  LZ5_loadDictHC (LZ5_streamHC_t* streamHCPtr, const char* dictionary, int dictSize);
+
+int LZ5_compress_HC_continue (LZ5_streamHC_t* streamHCPtr, const char* src, char* dst, int srcSize, int maxDstSize);
+
+int LZ5_saveDictHC (LZ5_streamHC_t* streamHCPtr, char* safeBuffer, int maxDictSize);
+
+/*
+  These functions compress data in successive blocks of any size, using previous blocks as dictionary.
+  One key assumption is that previous blocks (up to 64 KB) remain read-accessible while compressing next blocks.
+  There is an exception for ring buffers, which can be smaller 64 KB.
+  Such case is automatically detected and correctly handled by LZ5_compress_HC_continue().
+
+  Before starting compression, state must be properly initialized, using LZ5_resetStreamHC().
+  A first "fictional block" can then be designated as initial dictionary, using LZ5_loadDictHC() (Optional).
+
+  Then, use LZ5_compress_HC_continue() to compress each successive block.
+  It works like LZ5_compress_HC(), but use previous memory blocks as dictionary to improve compression.
+  Previous memory blocks (including initial dictionary when present) must remain accessible and unmodified during compression.
+  As a reminder, size 'dst' buffer to handle worst cases, using LZ5_compressBound(), to ensure success of compression operation.
+
+  If, for any reason, previous data blocks can't be preserved unmodified in memory during next compression block,
+  you must save it to a safer memory space, using LZ5_saveDictHC().
+  Return value of LZ5_saveDictHC() is the size of dictionary effectively saved into 'safeBuffer'.
+*/
+
+
+
+/**************************************
+*  Deprecated Functions
+**************************************/
+/* Deprecate Warnings */
+/* Should these warnings messages be a problem,
+   it is generally possible to disable them,
+   with -Wno-deprecated-declarations for gcc
+   or _CRT_SECURE_NO_WARNINGS in Visual for example.
+   You can also define LZ5_DEPRECATE_WARNING_DEFBLOCK. */
+#ifndef LZ5_DEPRECATE_WARNING_DEFBLOCK
+#  define LZ5_DEPRECATE_WARNING_DEFBLOCK
+#  if (LZ5_GCC_VERSION >= 405) || defined(__clang__)
+#    define LZ5_DEPRECATED(message) __attribute__((deprecated(message)))
+#  elif (LZ5_GCC_VERSION >= 301)
+#    define LZ5_DEPRECATED(message) __attribute__((deprecated))
+#  elif defined(_MSC_VER)
+#    define LZ5_DEPRECATED(message) __declspec(deprecated(message))
+#  else
+#    pragma message("WARNING: You need to implement LZ5_DEPRECATED for this compiler")
+#    define LZ5_DEPRECATED(message)
+#  endif
+#endif // LZ5_DEPRECATE_WARNING_DEFBLOCK
+
+/* compression functions */
+/* these functions are planned to trigger warning messages by r132 approximately */
+int LZ5_compressHC                (const char* source, char* dest, int inputSize);
+int LZ5_compressHC_limitedOutput  (const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ5_compressHC_continue               (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize);
+int LZ5_compressHC_limitedOutput_continue (LZ5_streamHC_t* LZ5_streamHCPtr, const char* source, char* dest, int inputSize, int maxOutputSize);
+int LZ5_compressHC_withStateHC               (void* state, const char* source, char* dest, int inputSize);
+int LZ5_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize); 
+
+#if defined (__cplusplus)
+}
+#endif

C/lz5/lz5xxhash.c

@@ -0,0 +1,915 @@
+/*
+xxHash - Fast Hash algorithm
+Copyright (C) 2012-2015, Yann Collet
+
+BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+* Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+You can contact the author at :
+- xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+
+/**************************************
+*  Tuning parameters
+**************************************/
+/* Unaligned memory access is automatically enabled for "common" CPU, such as x86.
+ * For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected.
+ * If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance.
+ * You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32).
+ */
+#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64)
+#  define XXH_USE_UNALIGNED_ACCESS 1
+#endif
+
+/* XXH_ACCEPT_NULL_INPUT_POINTER :
+ * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer.
+ * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input.
+ * By default, this option is disabled. To enable it, uncomment below define :
+ */
+/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */
+
+/* XXH_FORCE_NATIVE_FORMAT :
+ * By default, xxHash library provides endian-independant Hash values, based on little-endian convention.
+ * Results are therefore identical for little-endian and big-endian CPU.
+ * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format.
+ * Should endian-independance be of no importance for your application, you may set the #define below to 1.
+ * It will improve speed for Big-endian CPU.
+ * This option has no impact on Little_Endian CPU.
+ */
+#define XXH_FORCE_NATIVE_FORMAT 0
+
+
+/**************************************
+*  Compiler Specific Options
+***************************************/
+#ifdef _MSC_VER    /* Visual Studio */
+#  pragma warning(disable : 4127)      /* disable: C4127: conditional expression is constant */
+#  define FORCE_INLINE static __forceinline
+#else
+#  if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+#    ifdef __GNUC__
+#      define FORCE_INLINE static inline __attribute__((always_inline))
+#    else
+#      define FORCE_INLINE static inline
+#    endif
+#  else
+#    define FORCE_INLINE static
+#  endif /* __STDC_VERSION__ */
+#endif
+
+
+/**************************************
+*  Includes & Memory related functions
+***************************************/
+#include "xxhash.h"
+/* Modify the local functions below should you wish to use some other memory routines */
+/* for malloc(), free() */
+#include <stdlib.h>
+static void* XXH_malloc(size_t s) { return malloc(s); }
+static void  XXH_free  (void* p)  { free(p); }
+/* for memcpy() */
+#include <string.h>
+static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); }
+
+
+/**************************************
+*  Basic Types
+***************************************/
+#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L   /* C99 */
+# include <stdint.h>
+  typedef uint8_t  BYTE;
+  typedef uint16_t U16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+#else
+  typedef unsigned char      BYTE;
+  typedef unsigned short     U16;
+  typedef unsigned int       U32;
+  typedef   signed int       S32;
+  typedef unsigned long long U64;
+#endif
+
+static U32 XXH_read32(const void* memPtr)
+{
+    U32 val32;
+    memcpy(&val32, memPtr, 4);
+    return val32;
+}
+
+static U64 XXH_read64(const void* memPtr)
+{
+    U64 val64;
+    memcpy(&val64, memPtr, 8);
+    return val64;
+}
+
+
+
+/******************************************
+*  Compiler-specific Functions and Macros
+******************************************/
+#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
+
+/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */
+#if defined(_MSC_VER)
+#  define XXH_rotl32(x,r) _rotl(x,r)
+#  define XXH_rotl64(x,r) _rotl64(x,r)
+#else
+#  define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r)))
+#  define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r)))
+#endif
+
+#if defined(_MSC_VER)     /* Visual Studio */
+#  define XXH_swap32 _byteswap_ulong
+#  define XXH_swap64 _byteswap_uint64
+#elif GCC_VERSION >= 403
+#  define XXH_swap32 __builtin_bswap32
+#  define XXH_swap64 __builtin_bswap64
+#else
+static U32 XXH_swap32 (U32 x)
+{
+    return  ((x << 24) & 0xff000000 ) |
+            ((x <<  8) & 0x00ff0000 ) |
+            ((x >>  8) & 0x0000ff00 ) |
+            ((x >> 24) & 0x000000ff );
+}
+static U64 XXH_swap64 (U64 x)
+{
+    return  ((x << 56) & 0xff00000000000000ULL) |
+            ((x << 40) & 0x00ff000000000000ULL) |
+            ((x << 24) & 0x0000ff0000000000ULL) |
+            ((x << 8)  & 0x000000ff00000000ULL) |
+            ((x >> 8)  & 0x00000000ff000000ULL) |
+            ((x >> 24) & 0x0000000000ff0000ULL) |
+            ((x >> 40) & 0x000000000000ff00ULL) |
+            ((x >> 56) & 0x00000000000000ffULL);
+}
+#endif
+
+
+/***************************************
+*  Architecture Macros
+***************************************/
+typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess;
+#ifndef XXH_CPU_LITTLE_ENDIAN   /* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example using a compiler switch */
+static const int one = 1;
+#   define XXH_CPU_LITTLE_ENDIAN   (*(const char*)(&one))
+#endif
+
+
+/*****************************
+*  Memory reads
+*****************************/
+typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment;
+
+FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr);
+}
+
+FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE32_align(ptr, endian, XXH_unaligned);
+}
+
+FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
+{
+    if (align==XXH_unaligned)
+        return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
+    else
+        return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr);
+}
+
+FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian)
+{
+    return XXH_readLE64_align(ptr, endian, XXH_unaligned);
+}
+
+
+/***************************************
+*  Macros
+***************************************/
+#define XXH_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(!!(c)) }; }    /* use only *after* variable declarations */
+
+
+/***************************************
+*  Constants
+***************************************/
+#define PRIME32_1   2654435761U
+#define PRIME32_2   2246822519U
+#define PRIME32_3   3266489917U
+#define PRIME32_4    668265263U
+#define PRIME32_5    374761393U
+
+#define PRIME64_1 11400714785074694791ULL
+#define PRIME64_2 14029467366897019727ULL
+#define PRIME64_3  1609587929392839161ULL
+#define PRIME64_4  9650029242287828579ULL
+#define PRIME64_5  2870177450012600261ULL
+
+
+/*****************************
+*  Simple Hash Functions
+*****************************/
+FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U32 h32;
+#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL)
+    {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)16;
+    }
+#endif
+
+    if (len>=16)
+    {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = seed + PRIME32_1 + PRIME32_2;
+        U32 v2 = seed + PRIME32_2;
+        U32 v3 = seed + 0;
+        U32 v4 = seed - PRIME32_1;
+
+        do
+        {
+            v1 += XXH_get32bits(p) * PRIME32_2;
+            v1 = XXH_rotl32(v1, 13);
+            v1 *= PRIME32_1;
+            p+=4;
+            v2 += XXH_get32bits(p) * PRIME32_2;
+            v2 = XXH_rotl32(v2, 13);
+            v2 *= PRIME32_1;
+            p+=4;
+            v3 += XXH_get32bits(p) * PRIME32_2;
+            v3 = XXH_rotl32(v3, 13);
+            v3 *= PRIME32_1;
+            p+=4;
+            v4 += XXH_get32bits(p) * PRIME32_2;
+            v4 = XXH_rotl32(v4, 13);
+            v4 *= PRIME32_1;
+            p+=4;
+        }
+        while (p<=limit);
+
+        h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
+    }
+    else
+    {
+        h32  = seed + PRIME32_5;
+    }
+
+    h32 += (U32) len;
+
+    while (p+4<=bEnd)
+    {
+        h32 += XXH_get32bits(p) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4 ;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h32 += (*p) * PRIME32_5;
+        h32 = XXH_rotl32(h32, 11) * PRIME32_1 ;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+unsigned XXH32 (const void* input, size_t len, unsigned seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH32_state_t state;
+    XXH32_reset(&state, seed);
+    XXH32_update(&state, input, len);
+    return XXH32_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+#  if !defined(XXH_USE_UNALIGNED_ACCESS)
+    if ((((size_t)input) & 3) == 0)   /* Input is 4-bytes aligned, leverage the speed benefit */
+    {
+        if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+            return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+        else
+            return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }
+#  endif
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align)
+{
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* bEnd = p + len;
+    U64 h64;
+#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align)
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (p==NULL)
+    {
+        len=0;
+        bEnd=p=(const BYTE*)(size_t)32;
+    }
+#endif
+
+    if (len>=32)
+    {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = seed + PRIME64_1 + PRIME64_2;
+        U64 v2 = seed + PRIME64_2;
+        U64 v3 = seed + 0;
+        U64 v4 = seed - PRIME64_1;
+
+        do
+        {
+            v1 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v1 = XXH_rotl64(v1, 31);
+            v1 *= PRIME64_1;
+            v2 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v2 = XXH_rotl64(v2, 31);
+            v2 *= PRIME64_1;
+            v3 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v3 = XXH_rotl64(v3, 31);
+            v3 *= PRIME64_1;
+            v4 += XXH_get64bits(p) * PRIME64_2;
+            p+=8;
+            v4 = XXH_rotl64(v4, 31);
+            v4 *= PRIME64_1;
+        }
+        while (p<=limit);
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+
+        v1 *= PRIME64_2;
+        v1 = XXH_rotl64(v1, 31);
+        v1 *= PRIME64_1;
+        h64 ^= v1;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+
+        v2 *= PRIME64_2;
+        v2 = XXH_rotl64(v2, 31);
+        v2 *= PRIME64_1;
+        h64 ^= v2;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+
+        v3 *= PRIME64_2;
+        v3 = XXH_rotl64(v3, 31);
+        v3 *= PRIME64_1;
+        h64 ^= v3;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+
+        v4 *= PRIME64_2;
+        v4 = XXH_rotl64(v4, 31);
+        v4 *= PRIME64_1;
+        h64 ^= v4;
+        h64 = h64 * PRIME64_1 + PRIME64_4;
+    }
+    else
+    {
+        h64  = seed + PRIME64_5;
+    }
+
+    h64 += (U64) len;
+
+    while (p+8<=bEnd)
+    {
+        U64 k1 = XXH_get64bits(p);
+        k1 *= PRIME64_2;
+        k1 = XXH_rotl64(k1,31);
+        k1 *= PRIME64_1;
+        h64 ^= k1;
+        h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd)
+    {
+        h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1;
+        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h64 ^= (*p) * PRIME64_5;
+        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+
+unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed)
+{
+#if 0
+    /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
+    XXH64_state_t state;
+    XXH64_reset(&state, seed);
+    XXH64_update(&state, input, len);
+    return XXH64_digest(&state);
+#else
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+#  if !defined(XXH_USE_UNALIGNED_ACCESS)
+    if ((((size_t)input) & 7)==0)   /* Input is aligned, let's leverage the speed advantage */
+    {
+        if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+            return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned);
+        else
+            return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned);
+    }
+#  endif
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned);
+    else
+        return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned);
+#endif
+}
+
+/****************************************************
+*  Advanced Hash Functions
+****************************************************/
+
+/*** Allocation ***/
+typedef struct
+{
+    U64 total_len;
+    U32 seed;
+    U32 v1;
+    U32 v2;
+    U32 v3;
+    U32 v4;
+    U32 mem32[4];   /* defined as U32 for alignment */
+    U32 memsize;
+} XXH_istate32_t;
+
+typedef struct
+{
+    U64 total_len;
+    U64 seed;
+    U64 v1;
+    U64 v2;
+    U64 v3;
+    U64 v4;
+    U64 mem64[4];   /* defined as U64 for alignment */
+    U32 memsize;
+} XXH_istate64_t;
+
+
+XXH32_state_t* XXH32_createState(void)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH32_state_t) >= sizeof(XXH_istate32_t));   /* A compilation error here means XXH32_state_t is not large enough */
+    return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
+}
+XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+XXH64_state_t* XXH64_createState(void)
+{
+    XXH_STATIC_ASSERT(sizeof(XXH64_state_t) >= sizeof(XXH_istate64_t));   /* A compilation error here means XXH64_state_t is not large enough */
+    return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
+}
+XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
+{
+    XXH_free(statePtr);
+    return XXH_OK;
+}
+
+
+/*** Hash feed ***/
+
+XXH_errorcode XXH32_reset(XXH32_state_t* state_in, U32 seed)
+{
+    XXH_istate32_t* state = (XXH_istate32_t*) state_in;
+    state->seed = seed;
+    state->v1 = seed + PRIME32_1 + PRIME32_2;
+    state->v2 = seed + PRIME32_2;
+    state->v3 = seed + 0;
+    state->v4 = seed - PRIME32_1;
+    state->total_len = 0;
+    state->memsize = 0;
+    return XXH_OK;
+}
+
+XXH_errorcode XXH64_reset(XXH64_state_t* state_in, unsigned long long seed)
+{
+    XXH_istate64_t* state = (XXH_istate64_t*) state_in;
+    state->seed = seed;
+    state->v1 = seed + PRIME64_1 + PRIME64_2;
+    state->v2 = seed + PRIME64_2;
+    state->v3 = seed + 0;
+    state->v4 = seed - PRIME64_1;
+    state->total_len = 0;
+    state->memsize = 0;
+    return XXH_OK;
+}
+
+
+FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
+{
+    XXH_istate32_t* state = (XXH_istate32_t *) state_in;
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 16)   /* fill in tmp buffer */
+    {
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize)   /* some data left from previous update */
+    {
+        XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize);
+        {
+            const U32* p32 = state->mem32;
+            state->v1 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v1 = XXH_rotl32(state->v1, 13);
+            state->v1 *= PRIME32_1;
+            p32++;
+            state->v2 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v2 = XXH_rotl32(state->v2, 13);
+            state->v2 *= PRIME32_1;
+            p32++;
+            state->v3 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v3 = XXH_rotl32(state->v3, 13);
+            state->v3 *= PRIME32_1;
+            p32++;
+            state->v4 += XXH_readLE32(p32, endian) * PRIME32_2;
+            state->v4 = XXH_rotl32(state->v4, 13);
+            state->v4 *= PRIME32_1;
+            p32++;
+        }
+        p += 16-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p <= bEnd-16)
+    {
+        const BYTE* const limit = bEnd - 16;
+        U32 v1 = state->v1;
+        U32 v2 = state->v2;
+        U32 v3 = state->v3;
+        U32 v4 = state->v4;
+
+        do
+        {
+            v1 += XXH_readLE32(p, endian) * PRIME32_2;
+            v1 = XXH_rotl32(v1, 13);
+            v1 *= PRIME32_1;
+            p+=4;
+            v2 += XXH_readLE32(p, endian) * PRIME32_2;
+            v2 = XXH_rotl32(v2, 13);
+            v2 *= PRIME32_1;
+            p+=4;
+            v3 += XXH_readLE32(p, endian) * PRIME32_2;
+            v3 = XXH_rotl32(v3, 13);
+            v3 *= PRIME32_1;
+            p+=4;
+            v4 += XXH_readLE32(p, endian) * PRIME32_2;
+            v4 = XXH_rotl32(v4, 13);
+            v4 *= PRIME32_1;
+            p+=4;
+        }
+        while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd)
+    {
+        XXH_memcpy(state->mem32, p, bEnd-p);
+        state->memsize = (int)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH32_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state_in, XXH_endianess endian)
+{
+    const XXH_istate32_t* state = (const XXH_istate32_t*) state_in;
+    const BYTE * p = (const BYTE*)state->mem32;
+    const BYTE* bEnd = (const BYTE*)(state->mem32) + state->memsize;
+    U32 h32;
+
+    if (state->total_len >= 16)
+    {
+        h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18);
+    }
+    else
+    {
+        h32  = state->seed + PRIME32_5;
+    }
+
+    h32 += (U32) state->total_len;
+
+    while (p+4<=bEnd)
+    {
+        h32 += XXH_readLE32(p, endian) * PRIME32_3;
+        h32  = XXH_rotl32(h32, 17) * PRIME32_4;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h32 += (*p) * PRIME32_5;
+        h32 = XXH_rotl32(h32, 11) * PRIME32_1;
+        p++;
+    }
+
+    h32 ^= h32 >> 15;
+    h32 *= PRIME32_2;
+    h32 ^= h32 >> 13;
+    h32 *= PRIME32_3;
+    h32 ^= h32 >> 16;
+
+    return h32;
+}
+
+
+U32 XXH32_digest (const XXH32_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH32_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH32_digest_endian(state_in, XXH_bigEndian);
+}
+
+
+FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state_in, const void* input, size_t len, XXH_endianess endian)
+{
+    XXH_istate64_t * state = (XXH_istate64_t *) state_in;
+    const BYTE* p = (const BYTE*)input;
+    const BYTE* const bEnd = p + len;
+
+#ifdef XXH_ACCEPT_NULL_INPUT_POINTER
+    if (input==NULL) return XXH_ERROR;
+#endif
+
+    state->total_len += len;
+
+    if (state->memsize + len < 32)   /* fill in tmp buffer */
+    {
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len);
+        state->memsize += (U32)len;
+        return XXH_OK;
+    }
+
+    if (state->memsize)   /* some data left from previous update */
+    {
+        XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize);
+        {
+            const U64* p64 = state->mem64;
+            state->v1 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v1 = XXH_rotl64(state->v1, 31);
+            state->v1 *= PRIME64_1;
+            p64++;
+            state->v2 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v2 = XXH_rotl64(state->v2, 31);
+            state->v2 *= PRIME64_1;
+            p64++;
+            state->v3 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v3 = XXH_rotl64(state->v3, 31);
+            state->v3 *= PRIME64_1;
+            p64++;
+            state->v4 += XXH_readLE64(p64, endian) * PRIME64_2;
+            state->v4 = XXH_rotl64(state->v4, 31);
+            state->v4 *= PRIME64_1;
+            p64++;
+        }
+        p += 32-state->memsize;
+        state->memsize = 0;
+    }
+
+    if (p+32 <= bEnd)
+    {
+        const BYTE* const limit = bEnd - 32;
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        do
+        {
+            v1 += XXH_readLE64(p, endian) * PRIME64_2;
+            v1 = XXH_rotl64(v1, 31);
+            v1 *= PRIME64_1;
+            p+=8;
+            v2 += XXH_readLE64(p, endian) * PRIME64_2;
+            v2 = XXH_rotl64(v2, 31);
+            v2 *= PRIME64_1;
+            p+=8;
+            v3 += XXH_readLE64(p, endian) * PRIME64_2;
+            v3 = XXH_rotl64(v3, 31);
+            v3 *= PRIME64_1;
+            p+=8;
+            v4 += XXH_readLE64(p, endian) * PRIME64_2;
+            v4 = XXH_rotl64(v4, 31);
+            v4 *= PRIME64_1;
+            p+=8;
+        }
+        while (p<=limit);
+
+        state->v1 = v1;
+        state->v2 = v2;
+        state->v3 = v3;
+        state->v4 = v4;
+    }
+
+    if (p < bEnd)
+    {
+        XXH_memcpy(state->mem64, p, bEnd-p);
+        state->memsize = (int)(bEnd-p);
+    }
+
+    return XXH_OK;
+}
+
+XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_update_endian(state_in, input, len, XXH_littleEndian);
+    else
+        return XXH64_update_endian(state_in, input, len, XXH_bigEndian);
+}
+
+
+
+FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state_in, XXH_endianess endian)
+{
+    const XXH_istate64_t * state = (const XXH_istate64_t *) state_in;
+    const BYTE * p = (const BYTE*)state->mem64;
+    const BYTE* bEnd = (const BYTE*)state->mem64 + state->memsize;
+    U64 h64;
+
+    if (state->total_len >= 32)
+    {
+        U64 v1 = state->v1;
+        U64 v2 = state->v2;
+        U64 v3 = state->v3;
+        U64 v4 = state->v4;
+
+        h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
+
+        v1 *= PRIME64_2;
+        v1 = XXH_rotl64(v1, 31);
+        v1 *= PRIME64_1;
+        h64 ^= v1;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+
+        v2 *= PRIME64_2;
+        v2 = XXH_rotl64(v2, 31);
+        v2 *= PRIME64_1;
+        h64 ^= v2;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+
+        v3 *= PRIME64_2;
+        v3 = XXH_rotl64(v3, 31);
+        v3 *= PRIME64_1;
+        h64 ^= v3;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+
+        v4 *= PRIME64_2;
+        v4 = XXH_rotl64(v4, 31);
+        v4 *= PRIME64_1;
+        h64 ^= v4;
+        h64 = h64*PRIME64_1 + PRIME64_4;
+    }
+    else
+    {
+        h64  = state->seed + PRIME64_5;
+    }
+
+    h64 += (U64) state->total_len;
+
+    while (p+8<=bEnd)
+    {
+        U64 k1 = XXH_readLE64(p, endian);
+        k1 *= PRIME64_2;
+        k1 = XXH_rotl64(k1,31);
+        k1 *= PRIME64_1;
+        h64 ^= k1;
+        h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4;
+        p+=8;
+    }
+
+    if (p+4<=bEnd)
+    {
+        h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1;
+        h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3;
+        p+=4;
+    }
+
+    while (p<bEnd)
+    {
+        h64 ^= (*p) * PRIME64_5;
+        h64 = XXH_rotl64(h64, 11) * PRIME64_1;
+        p++;
+    }
+
+    h64 ^= h64 >> 33;
+    h64 *= PRIME64_2;
+    h64 ^= h64 >> 29;
+    h64 *= PRIME64_3;
+    h64 ^= h64 >> 32;
+
+    return h64;
+}
+
+
+unsigned long long XXH64_digest (const XXH64_state_t* state_in)
+{
+    XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN;
+
+    if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT)
+        return XXH64_digest_endian(state_in, XXH_littleEndian);
+    else
+        return XXH64_digest_endian(state_in, XXH_bigEndian);
+}
+
+

C/lz5/mem.h

@@ -0,0 +1,452 @@
+/* ******************************************************************
+   mem.h
+   low-level memory access routines
+   Copyright (C) 2013-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+    You can contact the author at :
+    - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy
+    - Public forum : https://groups.google.com/forum/#!forum/lz4c
+****************************************************************** */
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/******************************************
+*  Includes
+******************************************/
+#include <stddef.h>    /* size_t, ptrdiff_t */
+#include <string.h>    /* memcpy */
+
+
+
+/******************************************
+*  Compiler-specific
+******************************************/
+#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#  define MEM_STATIC static inline
+#elif defined(_MSC_VER)
+#  define MEM_STATIC static __inline
+#elif defined(__GNUC__)
+#  define MEM_STATIC static __attribute__((unused))
+#else
+#  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+
+
+/****************************************************************
+*  Memory I/O
+*****************************************************************/
+/* MEM_FORCE_MEMORY_ACCESS
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method is portable but violate C standard.
+ *            It can generate buggy code on targets generating assembly depending on alignment.
+ *            But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#    define MEM_FORCE_MEMORY_ACCESS 2
+#  elif defined(__INTEL_COMPILER) || \
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+#    define MEM_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; }
+MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; }
+
+MEM_STATIC unsigned MEM_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+}
+
+#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
+
+/* violates C standard on structure alignment.
+Only use if no other choice to achieve best performance on target platform */
+MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
+MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
+MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
+
+#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign;
+
+MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
+
+#else
+
+/* default method, safe and standard.
+   can sometimes prove slower */
+
+MEM_STATIC U16 MEM_read16(const void* memPtr)
+{
+    U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U32 MEM_read32(const void* memPtr)
+{
+    U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U64 MEM_read64(const void* memPtr)
+{
+    U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value)
+{
+    memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write32(void* memPtr, U32 value)
+{
+    memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write64(void* memPtr, U64 value)
+{
+    memcpy(memPtr, &value, sizeof(value));
+}
+
+#endif // MEM_FORCE_MEMORY_ACCESS
+
+
+MEM_STATIC U16 MEM_readLE16(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read16(memPtr);
+    else
+    {
+        const BYTE* p = (const BYTE*)memPtr;
+        return (U16)(p[0] + (p[1]<<8));
+    }
+}
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
+{
+    if (MEM_isLittleEndian())
+    {
+        MEM_write16(memPtr, val);
+    }
+    else
+    {
+        BYTE* p = (BYTE*)memPtr;
+        p[0] = (BYTE)val;
+        p[1] = (BYTE)(val>>8);
+    }
+}
+
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+    {
+        U32 val32 = 0;
+        memcpy(&val32, memPtr, 3);
+        return val32;
+    }
+    else
+    {
+        const BYTE* p = (const BYTE*)memPtr;
+        return (U32)(p[0] + (p[1]<<8) + (p[2]<<16));
+    }
+} 
+
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 value)
+{
+    if (MEM_isLittleEndian())
+    {
+        memcpy(memPtr, &value, 3);
+    }
+    else
+    {
+        BYTE* p = (BYTE*)memPtr;
+        p[0] = (BYTE) value;
+        p[1] = (BYTE)(value>>8);
+        p[2] = (BYTE)(value>>16);
+    }
+}
+
+ 
+MEM_STATIC U32 MEM_readLE32(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read32(memPtr);
+    else
+    {
+        const BYTE* p = (const BYTE*)memPtr;
+        return (U32)((U32)p[0] + ((U32)p[1]<<8) + ((U32)p[2]<<16) + ((U32)p[3]<<24));
+    }
+}
+
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
+{
+    if (MEM_isLittleEndian())
+    {
+        MEM_write32(memPtr, val32);
+    }
+    else
+    {
+        BYTE* p = (BYTE*)memPtr;
+        p[0] = (BYTE)val32;
+        p[1] = (BYTE)(val32>>8);
+        p[2] = (BYTE)(val32>>16);
+        p[3] = (BYTE)(val32>>24);
+    }
+}
+
+MEM_STATIC U64 MEM_readLE64(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read64(memPtr);
+    else
+    {
+        const BYTE* p = (const BYTE*)memPtr;
+        return (U64)((U64)p[0] + ((U64)p[1]<<8) + ((U64)p[2]<<16) + ((U64)p[3]<<24)
+                     + ((U64)p[4]<<32) + ((U64)p[5]<<40) + ((U64)p[6]<<48) + ((U64)p[7]<<56));
+    }
+}
+
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
+{
+    if (MEM_isLittleEndian())
+    {
+        MEM_write64(memPtr, val64);
+    }
+    else
+    {
+        BYTE* p = (BYTE*)memPtr;
+        p[0] = (BYTE)val64;
+        p[1] = (BYTE)(val64>>8);
+        p[2] = (BYTE)(val64>>16);
+        p[3] = (BYTE)(val64>>24);
+        p[4] = (BYTE)(val64>>32);
+        p[5] = (BYTE)(val64>>40);
+        p[6] = (BYTE)(val64>>48);
+        p[7] = (BYTE)(val64>>56);
+    }
+}
+
+MEM_STATIC size_t MEM_readLEST(const void* memPtr)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_readLE32(memPtr);
+    else
+        return (size_t)MEM_readLE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
+{
+    if (MEM_32bits())
+        MEM_writeLE32(memPtr, (U32)val);
+    else
+        MEM_writeLE64(memPtr, (U64)val);
+}
+
+
+#if MINMATCH == 3
+    #define MEM_read24(ptr) (U32)(MEM_read32(ptr)<<8) 
+#else
+    #define MEM_read24(ptr) (U32)(MEM_read32(ptr)) 
+#endif
+
+
+/* **************************************
+*  Function body to include for inlining
+****************************************/
+static size_t MEM_read_ARCH(const void* p) { size_t r; memcpy(&r, p, sizeof(r)); return r; }
+
+#define MIN(a,b) ((a)<(b) ? (a) : (b))
+
+
+/*static unsigned MEM_highbit(U32 val)
+{
+#   if defined(_MSC_VER)   // Visual 
+    unsigned long r=0;
+    _BitScanReverse(&r, val);
+    return (unsigned)r;
+#   elif defined(__GNUC__) && (__GNUC__ >= 3)   // GCC Intrinsic
+    return 31 - __builtin_clz(val);
+#   else   // Software version 
+    static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
+    U32 v = val;
+    int r;
+    v |= v >> 1;
+    v |= v >> 2;
+    v |= v >> 4;
+    v |= v >> 8;
+    v |= v >> 16;
+    r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
+    return r;
+#   endif
+}*/
+
+
+MEM_STATIC unsigned MEM_NbCommonBytes (register size_t val)
+{
+    if (MEM_isLittleEndian())
+    {
+        if (MEM_64bits())
+        {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanForward64( &r, (U64)val );
+            return (int)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctzll((U64)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 };
+            return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
+#       endif
+        }
+        else /* 32 bits */
+        {
+#       if defined(_MSC_VER)
+            unsigned long r=0;
+            _BitScanForward( &r, (U32)val );
+            return (int)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_ctz((U32)val) >> 3);
+#       else
+            static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 };
+            return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
+#       endif
+        }
+    }
+    else   /* Big Endian CPU */
+    {
+        if (MEM_32bits())
+        {
+#       if defined(_MSC_VER) && defined(_WIN64)
+            unsigned long r = 0;
+            _BitScanReverse64( &r, val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clzll(val) >> 3);
+#       else
+            unsigned r;
+            const unsigned n32 = sizeof(size_t)*4;   /* calculate this way due to compiler complaining in 32-bits mode */
+            if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; }
+            if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+        }
+        else /* 32 bits */
+        {
+#       if defined(_MSC_VER)
+            unsigned long r = 0;
+            _BitScanReverse( &r, (unsigned long)val );
+            return (unsigned)(r>>3);
+#       elif defined(__GNUC__) && (__GNUC__ >= 3)
+            return (__builtin_clz((U32)val) >> 3);
+#       else
+            unsigned r;
+            if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; }
+            r += (!val);
+            return r;
+#       endif
+        }
+    }
+}
+
+
+MEM_STATIC size_t MEM_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* pInLimit)
+{
+    const BYTE* const pStart = pIn;
+
+    while ((pIn<pInLimit-(sizeof(size_t)-1)))
+    {
+        size_t diff = MEM_read_ARCH(pMatch) ^ MEM_read_ARCH(pIn);
+        if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; }
+        pIn += MEM_NbCommonBytes(diff);
+        return (size_t)(pIn - pStart);
+    }
+
+    if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; }
+    if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; }
+    if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
+    return (size_t)(pIn - pStart);
+}
+
+
+static void MEM_copy8(void* dst, const void* src) { memcpy(dst, src, 8); }
+
+#define COPY8(d,s) { MEM_copy8(d,s); d+=8; s+=8; }
+
+
+/*! MEM_wildcopy : custom version of memcpy(), can copy up to 7-8 bytes too many */
+/*static void MEM_wildcopy(void* dst, const void* src, size_t length)
+{
+    const BYTE* ip = (const BYTE*)src;
+    BYTE* op = (BYTE*)dst;
+    BYTE* const oend = op + length;
+    do
+        COPY8(op, ip)
+    while (op < oend);
+} */ 
+
+/* customized variant of memcpy, which can overwrite up to 7 bytes beyond dstEnd */
+static void MEM_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd)
+{
+    BYTE* d = (BYTE*)dstPtr;
+    const BYTE* s = (const BYTE*)srcPtr;
+    BYTE* const e = (BYTE*)dstEnd;
+
+    do { MEM_copy8(d,s); d+=8; s+=8; } while (d<e);
+}  
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* MEM_H_MODULE */
+

C/lz5/xxhash.h

@@ -0,0 +1,192 @@
+/*
+   xxHash - Extremely Fast Hash algorithm
+   Header File
+   Copyright (C) 2012-2015, Yann Collet.
+
+   BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
+
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions are
+   met:
+
+       * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+       * Redistributions in binary form must reproduce the above
+   copyright notice, this list of conditions and the following disclaimer
+   in the documentation and/or other materials provided with the
+   distribution.
+
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+   You can contact the author at :
+   - xxHash source repository : https://github.com/Cyan4973/xxHash
+*/
+
+/* Notice extracted from xxHash homepage :
+
+xxHash is an extremely fast Hash algorithm, running at RAM speed limits.
+It also successfully passes all tests from the SMHasher suite.
+
+Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz)
+
+Name            Speed       Q.Score   Author
+xxHash          5.4 GB/s     10
+CrapWow         3.2 GB/s      2       Andrew
+MumurHash 3a    2.7 GB/s     10       Austin Appleby
+SpookyHash      2.0 GB/s     10       Bob Jenkins
+SBox            1.4 GB/s      9       Bret Mulvey
+Lookup3         1.2 GB/s      9       Bob Jenkins
+SuperFastHash   1.2 GB/s      1       Paul Hsieh
+CityHash64      1.05 GB/s    10       Pike & Alakuijala
+FNV             0.55 GB/s     5       Fowler, Noll, Vo
+CRC32           0.43 GB/s     9
+MD5-32          0.33 GB/s    10       Ronald L. Rivest
+SHA1-32         0.28 GB/s    10
+
+Q.Score is a measure of quality of the hash function.
+It depends on successfully passing SMHasher test set.
+10 is a perfect score.
+
+A 64-bits version, named XXH64, is available since r35.
+It offers much better speed, but for 64-bits applications only.
+Name     Speed on 64 bits    Speed on 32 bits
+XXH64       13.8 GB/s            1.9 GB/s
+XXH32        6.8 GB/s            6.0 GB/s
+*/
+
+#pragma once
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+/*****************************
+*  Definitions
+*****************************/
+#include <stddef.h>   /* size_t */
+typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode;
+
+
+/*****************************
+*  Namespace Emulation
+*****************************/
+/* Motivations :
+
+If you need to include xxHash into your library,
+but wish to avoid xxHash symbols to be present on your library interface
+in an effort to avoid potential name collision if another library also includes xxHash,
+
+you can use XXH_NAMESPACE, which will automatically prefix any symbol from xxHash
+with the value of XXH_NAMESPACE (so avoid to keep it NULL, and avoid numeric values).
+
+Note that no change is required within the calling program :
+it can still call xxHash functions using their regular name.
+They will be automatically translated by this header.
+*/
+#ifdef XXH_NAMESPACE
+#  define XXH_CAT(A,B) A##B
+#  define XXH_NAME2(A,B) XXH_CAT(A,B)
+#  define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
+#  define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
+#  define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
+#  define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
+#  define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
+#  define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
+#  define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
+#  define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
+#  define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
+#  define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
+#  define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
+#  define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
+#endif
+
+
+/*****************************
+*  Simple Hash Functions
+*****************************/
+
+unsigned int       XXH32 (const void* input, size_t length, unsigned seed);
+unsigned long long XXH64 (const void* input, size_t length, unsigned long long seed);
+
+/*
+XXH32() :
+    Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input".
+    The memory between input & input+length must be valid (allocated and read-accessible).
+    "seed" can be used to alter the result predictably.
+    This function successfully passes all SMHasher tests.
+    Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s
+XXH64() :
+    Calculate the 64-bits hash of sequence of length "len" stored at memory address "input".
+    Faster on 64-bits systems. Slower on 32-bits systems.
+*/
+
+
+
+/*****************************
+*  Advanced Hash Functions
+*****************************/
+typedef struct { long long ll[ 6]; } XXH32_state_t;
+typedef struct { long long ll[11]; } XXH64_state_t;
+
+/*
+These structures allow static allocation of XXH states.
+States must then be initialized using XXHnn_reset() before first use.
+
+If you prefer dynamic allocation, please refer to functions below.
+*/
+
+XXH32_state_t* XXH32_createState(void);
+XXH_errorcode  XXH32_freeState(XXH32_state_t* statePtr);
+
+XXH64_state_t* XXH64_createState(void);
+XXH_errorcode  XXH64_freeState(XXH64_state_t* statePtr);
+
+/*
+These functions create and release memory for XXH state.
+States must then be initialized using XXHnn_reset() before first use.
+*/
+
+
+XXH_errorcode XXH32_reset  (XXH32_state_t* statePtr, unsigned seed);
+XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
+unsigned int  XXH32_digest (const XXH32_state_t* statePtr);
+
+XXH_errorcode      XXH64_reset  (XXH64_state_t* statePtr, unsigned long long seed);
+XXH_errorcode      XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length);
+unsigned long long XXH64_digest (const XXH64_state_t* statePtr);
+
+/*
+These functions calculate the xxHash of an input provided in multiple smaller packets,
+as opposed to an input provided as a single block.
+
+XXH state space must first be allocated, using either static or dynamic method provided above.
+
+Start a new hash by initializing state with a seed, using XXHnn_reset().
+
+Then, feed the hash state by calling XXHnn_update() as many times as necessary.
+Obviously, input must be valid, meaning allocated and read accessible.
+The function returns an error code, with 0 meaning OK, and any other value meaning there is an error.
+
+Finally, you can produce a hash anytime, by using XXHnn_digest().
+This function returns the final nn-bits hash.
+You can nonetheless continue feeding the hash state with more input,
+and therefore get some new hashes, by calling again XXHnn_digest().
+
+When you are done, don't forget to free XXH state space, using typically XXHnn_freeState().
+*/
+
+
+#if defined (__cplusplus)
+}
+#endif

C/zstd/error_private.c

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/* The purpose of this file is to have a single list of error strings embedded in binary */
+
+#include "error_private.h"
+
+const char* ERR_getErrorString(ERR_enum code)
+{
+    static const char* const notErrorCode = "Unspecified error code";
+    switch( code )
+    {
+    case PREFIX(no_error): return "No error detected";
+    case PREFIX(GENERIC):  return "Error (generic)";
+    case PREFIX(prefix_unknown): return "Unknown frame descriptor";
+    case PREFIX(version_unsupported): return "Version not supported";
+    case PREFIX(parameter_unknown): return "Unknown parameter type";
+    case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
+    case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
+    case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding";
+    case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
+    case PREFIX(init_missing): return "Context should be init first";
+    case PREFIX(memory_allocation): return "Allocation error : not enough memory";
+    case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
+    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
+    case PREFIX(srcSize_wrong): return "Src size incorrect";
+    case PREFIX(corruption_detected): return "Corrupted block detected";
+    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
+    case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
+    case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
+    case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
+    case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
+    case PREFIX(dictionary_wrong): return "Dictionary mismatch";
+    case PREFIX(maxCode):
+    default: return notErrorCode;
+    }
+}

C/zstd/error_private.h

@@ -21,7 +21,7 @@ extern "C" {
 *  Dependencies
 ******************************************/
 #include <stddef.h>        /* size_t */
-#include "error_public.h"  /* enum list */
+#include "zstd_errors.h"  /* enum list */
 
 
 /* ****************************************
@@ -62,35 +62,7 @@ ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) retu
 *  Error Strings
 ******************************************/
 
-ERR_STATIC const char* ERR_getErrorString(ERR_enum code)
+const char* ERR_getErrorString(ERR_enum code);   /* error_private.c */
-{
-    static const char* notErrorCode = "Unspecified error code";
-    switch( code )
-    {
-    case PREFIX(no_error): return "No error detected";
-    case PREFIX(GENERIC):  return "Error (generic)";
-    case PREFIX(prefix_unknown): return "Unknown frame descriptor";
-    case PREFIX(version_unsupported): return "Version not supported";
-    case PREFIX(parameter_unknown): return "Unknown parameter type";
-    case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter";
-    case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode";
-    case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound";
-    case PREFIX(init_missing): return "Context should be init first";
-    case PREFIX(memory_allocation): return "Allocation error : not enough memory";
-    case PREFIX(stage_wrong): return "Operation not authorized at current processing stage";
-    case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
-    case PREFIX(srcSize_wrong): return "Src size incorrect";
-    case PREFIX(corruption_detected): return "Corrupted block detected";
-    case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
-    case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported";
-    case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large";
-    case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small";
-    case PREFIX(dictionary_corrupted): return "Dictionary is corrupted";
-    case PREFIX(dictionary_wrong): return "Dictionary mismatch";
-    case PREFIX(maxCode):
-    default: return notErrorCode;
-    }
-}
 
 ERR_STATIC const char* ERR_getErrorName(size_t code)
 {

C/zstd/fse.h

@@ -503,6 +503,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt
     BIT_flushBits(bitC);
 }
 
+
 /* ======    Decompression    ====== */
 
 typedef struct {
@@ -581,14 +582,19 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr)
 *  Increasing memory usage improves compression ratio
 *  Reduced memory usage can improve speed, due to cache effect
 *  Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */
-#define FSE_MAX_MEMORY_USAGE 14
+#ifndef FSE_MAX_MEMORY_USAGE
-#define FSE_DEFAULT_MEMORY_USAGE 13
+#  define FSE_MAX_MEMORY_USAGE 14
+#endif
+#ifndef FSE_DEFAULT_MEMORY_USAGE
+#  define FSE_DEFAULT_MEMORY_USAGE 13
+#endif
 
 /*!FSE_MAX_SYMBOL_VALUE :
 *  Maximum symbol value authorized.
 *  Required for proper stack allocation */
-#define FSE_MAX_SYMBOL_VALUE 255
+#ifndef FSE_MAX_SYMBOL_VALUE
-
+#  define FSE_MAX_SYMBOL_VALUE 255
+#endif
 
 /* **************************************************************
 *  template functions type & suffix

C/zstd/zstd.h

@@ -33,7 +33,7 @@ extern "C" {
 /*=======   Version   =======*/
 #define ZSTD_VERSION_MAJOR    1
 #define ZSTD_VERSION_MINOR    1
-#define ZSTD_VERSION_RELEASE  0
+#define ZSTD_VERSION_RELEASE  1
 
 #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE
 #define ZSTD_QUOTE(str) #str

C/zstd/zstd_compress.c

@@ -1458,7 +1458,7 @@ static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, co
     const U32 dictLimit = zc->dictLimit;
     const BYTE* const dictEnd = dictBase + dictLimit;
     const BYTE* const prefixStart = base + dictLimit;
-    const BYTE* match = base + matchIndex;
+    const BYTE* match;
     const U32 current = (U32)(ip-base);
     const U32 btLow = btMask >= current ? 0 : current - btMask;
     U32* smallerPtr = bt + 2*(current&btMask);
@@ -2235,7 +2235,7 @@ static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx,
     BYTE* op = ostart;
     U32 const maxDist = 1 << cctx->params.cParams.windowLog;
 
-    if (cctx->params.fParams.checksumFlag)
+    if (cctx->params.fParams.checksumFlag && srcSize)
         XXH64_update(&cctx->xxhState, src, srcSize);
 
     while (remaining) {
@@ -2688,7 +2688,9 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_pa
             return NULL;
         }
 
+        if (dictSize) {
             memcpy(dictContent, dict, dictSize);
+        }
         {   size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0);
             if (ZSTD_isError(errorCode)) {
                 ZSTD_free(dictContent, customMem);

C/zstd/zstd_decompress.c

@@ -248,7 +248,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
         if (!singleSegment) {
             BYTE const wlByte = ip[pos++];
             U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN;
-            if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_unsupported);
+            if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge);  /* avoids issue with 1 << windowLog */
             windowSize = (1U << windowLog);
             windowSize += (windowSize >> 3) * (wlByte&7);
         }
@@ -270,7 +270,7 @@ size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t
             case 3 : frameContentSize = MEM_readLE64(ip+pos); break;
         }
         if (!windowSize) windowSize = (U32)frameContentSize;
-        if (windowSize > windowSizeMax) return ERROR(frameParameter_unsupported);
+        if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge);
         fparamsPtr->frameContentSize = frameContentSize;
         fparamsPtr->windowSize = windowSize;
         fparamsPtr->dictID = dictID;
@@ -878,7 +878,12 @@ size_t ZSTD_execSequence(BYTE* op,
             op = oLitEnd + length1;
             sequence.matchLength -= length1;
             match = base;
+            if (op > oend_w) {
+              memmove(op, match, sequence.matchLength);
+              return sequenceLength;
+            }
     }   }
+    /* Requirement: op <= oend_w */
 
     /* match within prefix */
     if (sequence.offset < 8) {
@@ -1397,7 +1402,9 @@ ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_cu
             return NULL;
         }
 
+        if (dictSize) {
             memcpy(dictContent, dict, dictSize);
+        }
         {   size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize);
             if (ZSTD_isError(errorCode)) {
                 ZSTD_free(dictContent, customMem);
@@ -1568,7 +1575,7 @@ size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds,
     switch(paramType)
     {
         default : return ERROR(parameter_unknown);
-        case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue; break;
+        case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break;
     }
     return 0;
 }
@@ -1649,7 +1656,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB
             }   }
 
             zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN);
-            if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_unsupported);
+            if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge);
 
             /* Adapt buffer sizes to frame header instructions */
             {   size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX);

C/zstd/zstd_errors.h

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef ZSTD_ERRORS_H_398273423
+#define ZSTD_ERRORS_H_398273423
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*===== dependency =====*/
+#include <stddef.h>   /* size_t */
+
+
+/*-****************************************
+*  error codes list
+******************************************/
+typedef enum {
+  ZSTD_error_no_error,
+  ZSTD_error_GENERIC,
+  ZSTD_error_prefix_unknown,
+  ZSTD_error_version_unsupported,
+  ZSTD_error_parameter_unknown,
+  ZSTD_error_frameParameter_unsupported,
+  ZSTD_error_frameParameter_unsupportedBy32bits,
+  ZSTD_error_frameParameter_windowTooLarge,
+  ZSTD_error_compressionParameter_unsupported,
+  ZSTD_error_init_missing,
+  ZSTD_error_memory_allocation,
+  ZSTD_error_stage_wrong,
+  ZSTD_error_dstSize_tooSmall,
+  ZSTD_error_srcSize_wrong,
+  ZSTD_error_corruption_detected,
+  ZSTD_error_checksum_wrong,
+  ZSTD_error_tableLog_tooLarge,
+  ZSTD_error_maxSymbolValue_tooLarge,
+  ZSTD_error_maxSymbolValue_tooSmall,
+  ZSTD_error_dictionary_corrupted,
+  ZSTD_error_dictionary_wrong,
+  ZSTD_error_maxCode
+} ZSTD_ErrorCode;
+
+/*! ZSTD_getErrorCode() :
+    convert a `size_t` function result into a `ZSTD_ErrorCode` enum type,
+    which can be used to compare directly with enum list published into "error_public.h" */
+ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult);
+const char* ZSTD_getErrorString(ZSTD_ErrorCode code);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* ZSTD_ERRORS_H_398273423 */

C/zstd/zstd_opt.h

@@ -401,7 +401,7 @@ void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx,
     ZSTD_rescaleFreqs(seqStorePtr);
     ip += (ip==prefixStart);
     { U32 i; for (i=0; i<ZSTD_REP_NUM; i++) rep[i]=ctx->rep[i]; }
-    inr = ip;
+    //inr = ip;
 
     /* Match Loop */
     while (ip < ilimit) {
@@ -657,7 +657,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
     ctx->nextToUpdate3 = ctx->nextToUpdate;
     ZSTD_rescaleFreqs(seqStorePtr);
     ip += (ip==prefixStart);
-    inr = ip;
+    //inr = ip;
 
     /* Match Loop */
     while (ip < ilimit) {
@@ -666,7 +666,7 @@ void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx,
         U32 current = (U32)(ip-base);
         memset(opt, 0, sizeof(ZSTD_optimal_t));
         last_pos = 0;
-        inr = ip;
+        //inr = ip;
         opt[0].litlen = (U32)(ip - anchor);
 
         /* check repCode */

C/zstd/zstd_v04.c

@@ -3107,8 +3107,13 @@ static size_t ZSTD_execSequence(BYTE* op,
             op = oLitEnd + length1;
             sequence.matchLength -= length1;
             match = base;
+            if (op > oend_8) {
+              memmove(op, match, sequence.matchLength);
+              return sequenceLength;
             }
         }
+    }
+    /* Requirement: op <= oend_8 */
 
     /* match within prefix */
     if (sequence.offset < 8)

C/zstd/zstd_v05.c

@@ -3312,7 +3312,12 @@ static size_t ZSTDv05_execSequence(BYTE* op,
             op = oLitEnd + length1;
             sequence.matchLength -= length1;
             match = base;
+            if (op > oend_8) {
+              memmove(op, match, sequence.matchLength);
+              return sequenceLength;
+            }
     }   }
+    /* Requirement: op <= oend_8 */
 
     /* match within prefix */
     if (sequence.offset < 8) {

C/zstd/zstd_v06.c

@@ -3466,7 +3466,12 @@ size_t ZSTDv06_execSequence(BYTE* op,
             op = oLitEnd + length1;
             sequence.matchLength -= length1;
             match = base;
+            if (op > oend_8) {
+              memmove(op, match, sequence.matchLength);
+              return sequenceLength;
+            }
     }   }
+    /* Requirement: op <= oend_8 */
 
     /* match within prefix */
     if (sequence.offset < 8) {

C/zstd/zstd_v07.c

@@ -3690,7 +3690,12 @@ size_t ZSTDv07_execSequence(BYTE* op,
             op = oLitEnd + length1;
             sequence.matchLength -= length1;
             match = base;
+            if (op > oend_w) {
+              memmove(op, match, sequence.matchLength);
+              return sequenceLength;
+            }
     }   }
+    /* Requirement: op <= oend_w */
 
     /* match within prefix */
     if (sequence.offset < 8) {

C/zstdmt/README.md

@@ -0,0 +1,66 @@
+
+# Multithreading Library for [LZ4], [LZ5] and [ZStandard]
+
+
+### Compression
+```
+typedef struct {
+	void *buf;		/* ptr to data */
+	size_t size;		/* current filled in buf */
+	size_t allocated;	/* length of buf */
+} LZ4MT_Buffer;
+
+/**
+ * reading and writing functions
+ * - you can use stdio functions or plain read/write ...
+ * - a sample is given in 7-Zip ZS or lz4mt.c
+ */
+typedef int (fn_read) (void *args, LZ4MT_Buffer * in);
+typedef int (fn_write) (void *args, LZ4MT_Buffer * out);
+
+typedef struct {
+	fn_read *fn_read;
+	void *arg_read;
+	fn_write *fn_write;
+	void *arg_write;
+} LZ4MT_RdWr_t;
+
+typedef struct LZ4MT_CCtx_s LZ4MT_CCtx;
+
+/* 1) allocate new cctx */
+LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize);
+
+/* 2) threaded compression */
+size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr);
+
+/* 3) get some statistic */
+size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx);
+size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx);
+size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx);
+
+/* 4) free cctx */
+void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx);
+```
+
+### Decompression
+```
+typedef struct LZ4MT_DCtx_s LZ4MT_DCtx;
+
+/* 1) allocate new cctx */
+LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize);
+
+/* 2) threaded compression */
+size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr);
+
+/* 3) get some statistic */
+size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx);
+size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx);
+size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx);
+
+/* 4) free cctx */
+void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx);
+```
+
+## Todo
+
+- add Makefile

C/zstdmt/list.h

@@ -0,0 +1,130 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * - removed unneeded stuff
+ * - added win32 compatibility
+ *
+ * Copyright (c) 2013 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Taylor R. Campbell.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef LIST_H
+#define LIST_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include "mem.h"
+
+struct list_head {
+	struct list_head *prev;
+	struct list_head *next;
+};
+
+MEM_STATIC void INIT_LIST_HEAD(struct list_head *head)
+{
+	head->prev = head;
+	head->next = head;
+}
+
+MEM_STATIC struct list_head *list_first(const struct list_head *head)
+{
+	return head->next;
+}
+
+MEM_STATIC struct list_head *list_last(const struct list_head *head)
+{
+	return head->prev;
+}
+
+MEM_STATIC struct list_head *list_next(const struct list_head *node)
+{
+	return node->next;
+}
+
+MEM_STATIC struct list_head *list_prev(const struct list_head *node)
+{
+	return node->prev;
+}
+
+MEM_STATIC void __list_add_between(struct list_head *prev,
+	struct list_head *node, struct list_head *next)
+{
+	prev->next = node;
+	node->prev = prev;
+	node->next = next;
+	next->prev = node;
+}
+
+MEM_STATIC void list_add(struct list_head *node, struct list_head *head)
+{
+	__list_add_between(head, node, head->next);
+}
+
+MEM_STATIC void list_add_tail(struct list_head *node, struct list_head *head)
+{
+	__list_add_between(head->prev, node, head);
+}
+
+MEM_STATIC void list_del(struct list_head *entry)
+{
+	entry->prev->next = entry->next;
+	entry->next->prev = entry->prev;
+}
+
+MEM_STATIC int list_empty(const struct list_head *head)
+{
+	return (head->next == head);
+}
+
+MEM_STATIC void list_move(struct list_head *node, struct list_head *head)
+{
+	list_del(node);
+	list_add(node, head);
+}
+
+MEM_STATIC void list_move_tail(struct list_head *node, struct list_head *head)
+{
+	list_del(node);
+	list_add_tail(node, head);
+}
+
+#ifndef CONTAINING_RECORD
+#define CONTAINING_RECORD(ptr, type, field) \
+	((type*)((char*)(ptr) - (char*)(&((type*)0)->field)))
+#endif
+
+#define list_entry(ptr, type, member)	CONTAINING_RECORD(ptr, type, member)
+#define	list_for_each(var, head) \
+	for ((var) = list_first((head)); (var) != (head); (var) = list_next((var)))
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif				/* LIST_H */

C/zstdmt/lz4mt.h

@@ -0,0 +1,162 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/* ***************************************
+ * Defines
+ ****************************************/
+
+#ifndef LZ4MT_H
+#define LZ4MT_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <stddef.h>   /* size_t */
+
+/* current maximum the library will accept */
+#define LZ4MT_THREAD_MAX 128
+#define LZ4MT_LEVEL_MAX   16
+
+#define LZ4FMT_MAGICNUMBER     0x184D2204U
+#define LZ4FMT_MAGIC_SKIPPABLE 0x184D2A50U
+
+/* **************************************
+ * Error Handling
+ ****************************************/
+
+extern size_t lz4mt_errcode;
+
+typedef enum {
+  LZ4MT_error_no_error,
+  LZ4MT_error_memory_allocation,
+  LZ4MT_error_read_fail,
+  LZ4MT_error_write_fail,
+  LZ4MT_error_data_error,
+  LZ4MT_error_frame_compress,
+  LZ4MT_error_frame_decompress,
+  LZ4MT_error_compressionParameter_unsupported,
+  LZ4MT_error_compression_library,
+  LZ4MT_error_maxCode
+} LZ4MT_ErrorCode;
+
+#ifdef ERROR
+#  undef ERROR   /* reported already defined on VS 2015 (Rich Geldreich) */
+#endif
+#define PREFIX(name) LZ4MT_error_##name
+#define ERROR(name)  ((size_t)-PREFIX(name))
+extern unsigned LZ4MT_isError(size_t code);
+extern const char* LZ4MT_getErrorString(size_t code);
+
+/* **************************************
+ * Structures
+ ****************************************/
+
+typedef struct {
+	void *buf;		/* ptr to data */
+	size_t size;		/* current filled in buf */
+	size_t allocated;	/* length of buf */
+} LZ4MT_Buffer;
+
+/**
+ * reading and writing functions
+ * - you can use stdio functions or plain read/write
+ * - just write some wrapper on your own
+ * - a sample is given in 7-Zip ZS or lz4mt.c
+ * - the function should return -1 on error and zero on success
+ * - the read or written bytes will go to in->size or out->size
+ */
+typedef int (fn_read) (void *args, LZ4MT_Buffer * in);
+typedef int (fn_write) (void *args, LZ4MT_Buffer * out);
+
+typedef struct {
+	fn_read *fn_read;
+	void *arg_read;
+	fn_write *fn_write;
+	void *arg_write;
+} LZ4MT_RdWr_t;
+
+/* **************************************
+ * Compression
+ ****************************************/
+
+typedef struct LZ4MT_CCtx_s LZ4MT_CCtx;
+
+/**
+ * 1) allocate new cctx
+ * - return cctx or zero on error
+ *
+ * @level   - 1 .. 9
+ * @threads - 1 .. LZ4MT_THREAD_MAX
+ * @inputsize - if zero, becomes some optimal value for the level
+ *            - if nonzero, the given value is taken
+ */
+LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize);
+
+/**
+ * 2) threaded compression
+ * - errorcheck via 
+ */
+size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr);
+
+/**
+ * 3) get some statistic
+ */
+size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx);
+size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx);
+size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx);
+
+/**
+ * 4) free cctx
+ * - no special return value
+ */
+void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx);
+
+/* **************************************
+ * Decompression
+ ****************************************/
+
+typedef struct LZ4MT_DCtx_s LZ4MT_DCtx;
+
+/**
+ * 1) allocate new cctx
+ * - return cctx or zero on error
+ *
+ * @threads - 1 .. LZ4MT_THREAD_MAX
+ * @ inputsize - used for single threaded standard lz4 format without skippable frames
+ */
+LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize);
+
+/**
+ * 2) threaded compression
+ * - return -1 on error
+ */
+size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr);
+
+/**
+ * 3) get some statistic
+ */
+size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx);
+size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx);
+size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx);
+
+/**
+ * 4) free cctx
+ * - no special return value
+ */
+void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx);
+
+#if defined (__cplusplus)
+}
+#endif
+#endif				/* LZ4MT_H */

C/zstdmt/lz4mt_common.c

@@ -0,0 +1,62 @@
+
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include "lz4frame.h"
+#include "lz4mt.h"
+
+/* will be used for lib errors */
+size_t lz4mt_errcode;
+
+/* ****************************************
+ * LZ4MT Error Management
+ ******************************************/
+
+/**
+ * LZ4MT_isError() - tells if a return value is an error code
+ */
+unsigned LZ4MT_isError(size_t code)
+{
+	return (code > ERROR(maxCode));
+}
+
+/**
+ * LZ4MT_getErrorString() - give error code string from function result
+ */
+const char *LZ4MT_getErrorString(size_t code)
+{
+	if (LZ4F_isError(lz4mt_errcode))
+		return LZ4F_getErrorName(lz4mt_errcode);
+
+	static const char *notErrorCode = "Unspecified error lz4mt code";
+	switch ((LZ4MT_ErrorCode)(0-code)) {
+	case PREFIX(no_error):
+		return "No error detected";
+	case PREFIX(memory_allocation):
+		return "Allocation error : not enough memory";
+	case PREFIX(read_fail):
+		return "Read failure";
+	case PREFIX(write_fail):
+		return "Write failure";
+	case PREFIX(data_error):
+		return "Malformed input";
+	case PREFIX(frame_compress):
+		return "Could not compress frame at once";
+	case PREFIX(frame_decompress):
+		return "Could not decompress frame at once";
+	case PREFIX(compressionParameter_unsupported):
+		return "Compression parameter is out of bound";
+	case PREFIX(compression_library):
+		return "Compression library reports failure";
+	case PREFIX(maxCode):
+	default:
+		return notErrorCode;
+	}
+}

C/zstdmt/lz4mt_compress.c

@@ -0,0 +1,377 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#define LZ4F_DISABLE_OBSOLETE_ENUMS
+#include "lz4frame.h"
+
+#include "mem.h"
+#include "threading.h"
+#include "list.h"
+#include "lz4mt.h"
+
+/**
+ * multi threaded lz4 - multiple workers version
+ *
+ * - each thread works on his own
+ * - no main thread which does reading and then starting the work
+ * - needs a callback for reading / writing
+ * - each worker does his:
+ *   1) get read mutex and read some input
+ *   2) release read mutex and do compression
+ *   3) get write mutex and write result
+ *   4) begin with step 1 again, until no input
+ */
+
+/* worker for compression */
+typedef struct {
+	LZ4MT_CCtx *ctx;
+	LZ4F_preferences_t zpref;
+	pthread_t pthread;
+} cwork_t;
+
+struct writelist;
+struct writelist {
+	size_t frame;
+	LZ4MT_Buffer out;
+	struct list_head node;
+};
+
+struct LZ4MT_CCtx_s {
+
+	/* level: 1..22 */
+	int level;
+
+	/* threads: 1..LZ4MT_THREAD_MAX */
+	int threads;
+
+	/* should be used for read from input */
+	int inputsize;
+
+	/* statistic */
+	size_t insize;
+	size_t outsize;
+	size_t curframe;
+	size_t frames;
+
+	/* threading */
+	cwork_t *cwork;
+
+	/* reading input */
+	pthread_mutex_t read_mutex;
+	fn_read *fn_read;
+	void *arg_read;
+
+	/* writing output */
+	pthread_mutex_t write_mutex;
+	fn_write *fn_write;
+	void *arg_write;
+
+	/* lists for writing queue */
+	struct list_head writelist_free;
+	struct list_head writelist_busy;
+	struct list_head writelist_done;
+};
+
+/* **************************************
+ * Compression
+ ****************************************/
+
+LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize)
+{
+	LZ4MT_CCtx *ctx;
+	int t;
+
+	/* allocate ctx */
+	ctx = (LZ4MT_CCtx *) malloc(sizeof(LZ4MT_CCtx));
+	if (!ctx)
+		return 0;
+
+	/* check threads value */
+	if (threads < 1 || threads > LZ4MT_THREAD_MAX)
+		return 0;
+
+	/* check level */
+	if (level < 1 || level > LZ4MT_LEVEL_MAX)
+		return 0;
+
+	/* calculate chunksize for one thread */
+	if (inputsize)
+		ctx->inputsize = inputsize;
+	else
+		ctx->inputsize = 1024 * 64;
+
+	/* setup ctx */
+	ctx->level = level;
+	ctx->threads = threads;
+	ctx->insize = 0;
+	ctx->outsize = 0;
+	ctx->frames = 0;
+	ctx->curframe = 0;
+
+	pthread_mutex_init(&ctx->read_mutex, NULL);
+	pthread_mutex_init(&ctx->write_mutex, NULL);
+
+	/* free -> busy -> out -> free -> ... */
+	INIT_LIST_HEAD(&ctx->writelist_free);	/* free, can be used */
+	INIT_LIST_HEAD(&ctx->writelist_busy);	/* busy */
+	INIT_LIST_HEAD(&ctx->writelist_done);	/* can be written */
+
+	ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
+	if (!ctx->cwork)
+		goto err_cwork;
+
+	for (t = 0; t < threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->ctx = ctx;
+
+		/* setup preferences for that thread */
+		memset(&w->zpref, 0, sizeof(LZ4F_preferences_t));
+		w->zpref.compressionLevel = level;
+		w->zpref.frameInfo.blockMode = LZ4F_blockLinked;
+		w->zpref.frameInfo.contentSize = 1;
+		w->zpref.frameInfo.contentChecksumFlag =
+		    LZ4F_contentChecksumEnabled;
+
+	}
+
+	return ctx;
+
+ err_cwork:
+	free(ctx);
+
+	return 0;
+}
+
+/**
+ * pt_write - queue for compressed output
+ */
+static size_t pt_write(LZ4MT_CCtx * ctx, struct writelist *wl)
+{
+	struct list_head *entry;
+
+	/* move the entry to the done list */
+	list_move(&wl->node, &ctx->writelist_done);
+
+	/* the entry isn't the currently needed, return...  */
+	if (wl->frame != ctx->curframe)
+		return 0;
+
+ again:
+	/* check, what can be written ... */
+	list_for_each(entry, &ctx->writelist_done) {
+		wl = list_entry(entry, struct writelist, node);
+		if (wl->frame == ctx->curframe) {
+			int rv = ctx->fn_write(ctx->arg_write, &wl->out);
+			if (rv == -1)
+				return ERROR(write_fail);
+			ctx->outsize += wl->out.size;
+			ctx->curframe++;
+			list_move(entry, &ctx->writelist_free);
+			goto again;
+		}
+	}
+
+	return 0;
+}
+
+static void *pt_compress(void *arg)
+{
+	cwork_t *w = (cwork_t *) arg;
+	LZ4MT_CCtx *ctx = w->ctx;
+	size_t result;
+	LZ4MT_Buffer in;
+
+	/* inbuf is constant */
+	in.size = ctx->inputsize;
+	in.buf = malloc(in.size);
+	if (!in.buf)
+		return (void *)ERROR(memory_allocation);
+
+	for (;;) {
+		struct list_head *entry;
+		struct writelist *wl;
+		int rv;
+
+		/* allocate space for new output */
+		pthread_mutex_lock(&ctx->write_mutex);
+		if (!list_empty(&ctx->writelist_free)) {
+			/* take unused entry */
+			entry = list_first(&ctx->writelist_free);
+			wl = list_entry(entry, struct writelist, node);
+			wl->out.size =
+			    LZ4F_compressFrameBound(ctx->inputsize,
+						    &w->zpref) + 12;
+			list_move(entry, &ctx->writelist_busy);
+		} else {
+			/* allocate new one */
+			wl = (struct writelist *)
+			    malloc(sizeof(struct writelist));
+			if (!wl) {
+				pthread_mutex_unlock(&ctx->write_mutex);
+				return (void *)ERROR(memory_allocation);
+			}
+			wl->out.size =
+			    LZ4F_compressFrameBound(ctx->inputsize,
+						    &w->zpref) + 12;;
+			wl->out.buf = malloc(wl->out.size);
+			if (!wl->out.buf) {
+				pthread_mutex_unlock(&ctx->write_mutex);
+				return (void *)ERROR(memory_allocation);
+			}
+			list_add(&wl->node, &ctx->writelist_busy);
+		}
+		pthread_mutex_unlock(&ctx->write_mutex);
+
+		/* read new input */
+		pthread_mutex_lock(&ctx->read_mutex);
+		in.size = ctx->inputsize;
+		rv = ctx->fn_read(ctx->arg_read, &in);
+		if (rv == -1) {
+			pthread_mutex_unlock(&ctx->read_mutex);
+			return (void *)ERROR(read_fail);
+		}
+
+		/* eof */
+		if (in.size == 0) {
+			free(in.buf);
+			pthread_mutex_unlock(&ctx->read_mutex);
+
+			pthread_mutex_lock(&ctx->write_mutex);
+			list_move(&wl->node, &ctx->writelist_free);
+			pthread_mutex_unlock(&ctx->write_mutex);
+
+			goto okay;
+		}
+		ctx->insize += in.size;
+		wl->frame = ctx->frames++;
+		pthread_mutex_unlock(&ctx->read_mutex);
+
+		/* compress whole frame */
+		result =
+		    LZ4F_compressFrame((unsigned char *)wl->out.buf + 12,
+				       wl->out.size - 12, in.buf, in.size,
+				       &w->zpref);
+		if (LZ4F_isError(result)) {
+			pthread_mutex_lock(&ctx->write_mutex);
+			list_move(&wl->node, &ctx->writelist_free);
+			pthread_mutex_unlock(&ctx->write_mutex);
+			/* user can lookup that code */
+			lz4mt_errcode = result;
+			return (void *)ERROR(compression_library);
+		}
+
+		/* write skippable frame */
+		MEM_writeLE32((unsigned char *)wl->out.buf + 0,
+			      LZ4FMT_MAGIC_SKIPPABLE);
+		MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4);
+		MEM_writeLE32((unsigned char *)wl->out.buf + 8,
+			      (U32)result);
+		wl->out.size = result + 12;
+
+		/* write result */
+		pthread_mutex_lock(&ctx->write_mutex);
+		result = pt_write(ctx, wl);
+		pthread_mutex_unlock(&ctx->write_mutex);
+		if (LZ4MT_isError(result))
+			return (void *)result;
+	}
+
+ okay:
+	return 0;
+}
+
+size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr)
+{
+	int t;
+
+	if (!ctx)
+		return ERROR(compressionParameter_unsupported);
+
+	/* init reading and writing functions */
+	ctx->fn_read = rdwr->fn_read;
+	ctx->fn_write = rdwr->fn_write;
+	ctx->arg_read = rdwr->arg_read;
+	ctx->arg_write = rdwr->arg_write;
+
+	/* start all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		pthread_create(&w->pthread, NULL, pt_compress, w);
+	}
+
+	/* wait for all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		void *p;
+		pthread_join(w->pthread, &p);
+		if (p)
+			return (size_t) p;
+	}
+
+	/* clean up lists */
+	while (!list_empty(&ctx->writelist_free)) {
+		struct writelist *wl;
+		struct list_head *entry;
+		entry = list_first(&ctx->writelist_free);
+		wl = list_entry(entry, struct writelist, node);
+		free(wl->out.buf);
+		list_del(&wl->node);
+		free(wl);
+	}
+
+	return 0;
+}
+
+/* returns current uncompressed data size */
+size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->insize;
+}
+
+/* returns the current compressed data size */
+size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->outsize;
+}
+
+/* returns the current compressed frames */
+size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->curframe;
+}
+
+void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx)
+{
+	if (!ctx)
+		return;
+
+	pthread_mutex_destroy(&ctx->read_mutex);
+	pthread_mutex_destroy(&ctx->write_mutex);
+	free(ctx->cwork);
+	free(ctx);
+	ctx = 0;
+
+	return;
+}

C/zstdmt/lz4mt_decompress.c

@@ -0,0 +1,587 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#define LZ4F_DISABLE_OBSOLETE_ENUMS
+#include "lz4frame.h"
+
+#include "mem.h"
+#include "threading.h"
+#include "list.h"
+#include "lz4mt.h"
+
+/**
+ * multi threaded lz4 - multiple workers version
+ *
+ * - each thread works on his own
+ * - no main thread which does reading and then starting the work
+ * - needs a callback for reading / writing
+ * - each worker does his:
+ *   1) get read mutex and read some input
+ *   2) release read mutex and do compression
+ *   3) get write mutex and write result
+ *   4) begin with step 1 again, until no input
+ */
+
+/* worker for compression */
+typedef struct {
+	LZ4MT_DCtx *ctx;
+	pthread_t pthread;
+	LZ4MT_Buffer in;
+	LZ4F_decompressionContext_t dctx;
+} cwork_t;
+
+struct writelist;
+struct writelist {
+	size_t frame;
+	LZ4MT_Buffer out;
+	struct list_head node;
+};
+
+struct LZ4MT_DCtx_s {
+
+	/* threads: 1..LZ4MT_THREAD_MAX */
+	int threads;
+
+	/* should be used for read from input */
+	size_t inputsize;
+
+	/* statistic */
+	size_t insize;
+	size_t outsize;
+	size_t curframe;
+	size_t frames;
+
+	/* threading */
+	cwork_t *cwork;
+
+	/* reading input */
+	pthread_mutex_t read_mutex;
+	fn_read *fn_read;
+	void *arg_read;
+
+	/* writing output */
+	pthread_mutex_t write_mutex;
+	fn_write *fn_write;
+	void *arg_write;
+
+	/* lists for writing queue */
+	struct list_head writelist_free;
+	struct list_head writelist_busy;
+	struct list_head writelist_done;
+};
+
+/* **************************************
+ * Decompression
+ ****************************************/
+
+LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize)
+{
+	LZ4MT_DCtx *ctx;
+	int t;
+
+	/* allocate ctx */
+	ctx = (LZ4MT_DCtx *) malloc(sizeof(LZ4MT_DCtx));
+	if (!ctx)
+		return 0;
+
+	/* check threads value */
+	if (threads < 1 || threads > LZ4MT_THREAD_MAX)
+		return 0;
+
+	/* setup ctx */
+	ctx->threads = threads;
+	ctx->insize = 0;
+	ctx->outsize = 0;
+	ctx->frames = 0;
+	ctx->curframe = 0;
+
+	/* will be used for single stream only */
+	if (inputsize)
+		ctx->inputsize = inputsize;
+	else
+		ctx->inputsize = 1024 * 64;	/* 64K buffer */
+
+	pthread_mutex_init(&ctx->read_mutex, NULL);
+	pthread_mutex_init(&ctx->write_mutex, NULL);
+
+	INIT_LIST_HEAD(&ctx->writelist_free);
+	INIT_LIST_HEAD(&ctx->writelist_busy);
+	INIT_LIST_HEAD(&ctx->writelist_done);
+
+	ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
+	if (!ctx->cwork)
+		goto err_cwork;
+
+	for (t = 0; t < threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->ctx = ctx;
+
+		/* setup thread work */
+		LZ4F_createDecompressionContext(&w->dctx, LZ4F_VERSION);
+	}
+
+	return ctx;
+
+ err_cwork:
+	free(ctx);
+
+	return 0;
+}
+
+/**
+ * pt_write - queue for decompressed output
+ */
+static size_t pt_write(LZ4MT_DCtx * ctx, struct writelist *wl)
+{
+	struct list_head *entry;
+
+	/* move the entry to the done list */
+	list_move(&wl->node, &ctx->writelist_done);
+ again:
+	/* check, what can be written ... */
+	list_for_each(entry, &ctx->writelist_done) {
+		wl = list_entry(entry, struct writelist, node);
+		if (wl->frame == ctx->curframe) {
+			int rv = ctx->fn_write(ctx->arg_write, &wl->out);
+			if (rv == -1)
+				return ERROR(write_fail);
+			ctx->outsize += wl->out.size;
+			ctx->curframe++;
+			list_move(entry, &ctx->writelist_free);
+			goto again;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * pt_read - read compressed output
+ */
+static size_t pt_read(LZ4MT_DCtx * ctx, LZ4MT_Buffer * in, size_t * frame)
+{
+	unsigned char hdrbuf[12];
+	LZ4MT_Buffer hdr;
+	int rv;
+
+	/* read skippable frame (8 or 12 bytes) */
+	pthread_mutex_lock(&ctx->read_mutex);
+
+	/* special case, first 4 bytes already read */
+	if (ctx->frames == 0) {
+		hdr.buf = hdrbuf + 4;
+		hdr.size = 8;
+		rv = ctx->fn_read(ctx->arg_read, &hdr);
+		if (rv == -1)
+			goto error_read;
+		if (hdr.size != 8)
+			goto error_read;
+		hdr.buf = hdrbuf;
+	} else {
+		hdr.buf = hdrbuf;
+		hdr.size = 12;
+		rv = ctx->fn_read(ctx->arg_read, &hdr);
+		if (rv == -1)
+			goto error_read;
+		/* eof reached ? */
+		if (hdr.size == 0) {
+			pthread_mutex_unlock(&ctx->read_mutex);
+			in->size = 0;
+			return 0;
+		}
+		if (hdr.size != 12)
+			goto error_read;
+		if (MEM_readLE32((unsigned char *)hdr.buf + 0) !=
+		    LZ4FMT_MAGIC_SKIPPABLE)
+			goto error_data;
+	}
+
+	/* check header data */
+	if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4)
+		goto error_data;
+
+	ctx->insize += 12;
+	/* read new inputsize */
+	{
+		size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
+		if (in->allocated < toRead) {
+			/* need bigger input buffer */
+			if (in->allocated)
+				in->buf = realloc(in->buf, toRead);
+			else
+				in->buf = malloc(toRead);
+			if (!in->buf)
+				goto error_nomem;
+			in->allocated = toRead;
+		}
+
+		in->size = toRead;
+		rv = ctx->fn_read(ctx->arg_read, in);
+		/* generic read failure! */
+		if (rv == -1)
+			goto error_read;
+		/* needed more bytes! */
+		if (in->size != toRead)
+			goto error_data;
+
+		ctx->insize += in->size;
+	}
+	*frame = ctx->frames++;
+	pthread_mutex_unlock(&ctx->read_mutex);
+
+	/* done, no error */
+	return 0;
+
+ error_data:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ERROR(data_error);
+ error_read:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ERROR(read_fail);
+ error_nomem:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ERROR(memory_allocation);
+}
+
+static void *pt_decompress(void *arg)
+{
+	cwork_t *w = (cwork_t *) arg;
+	LZ4MT_Buffer *in = &w->in;
+	LZ4MT_DCtx *ctx = w->ctx;
+	size_t result = 0;
+	struct writelist *wl;
+
+	for (;;) {
+		struct list_head *entry;
+		LZ4MT_Buffer *out;
+
+		/* allocate space for new output */
+		pthread_mutex_lock(&ctx->write_mutex);
+		if (!list_empty(&ctx->writelist_free)) {
+			/* take unused entry */
+			entry = list_first(&ctx->writelist_free);
+			wl = list_entry(entry, struct writelist, node);
+			list_move(entry, &ctx->writelist_busy);
+		} else {
+			/* allocate new one */
+			wl = (struct writelist *)
+			    malloc(sizeof(struct writelist));
+			if (!wl) {
+				result = ERROR(memory_allocation);
+				goto error_unlock;
+			}
+			wl->out.buf = 0;
+			wl->out.size = 0;
+			wl->out.allocated = 0;
+			list_add(&wl->node, &ctx->writelist_busy);
+		}
+		pthread_mutex_unlock(&ctx->write_mutex);
+		out = &wl->out;
+
+		/* zero should not happen here! */
+		result = pt_read(ctx, in, &wl->frame);
+		if (LZ4MT_isError(result)) {
+			list_move(&wl->node, &ctx->writelist_free);
+			goto error_lock;
+		}
+
+		if (in->size == 0)
+			break;
+
+		{
+			/* get frame size for output buffer */
+			unsigned char *src = (unsigned char *)in->buf + 6;
+			out->size = (size_t) MEM_readLE64(src);
+		}
+
+		if (out->allocated < out->size) {
+			if (out->allocated)
+				out->buf = realloc(out->buf, out->size);
+			else
+				out->buf = malloc(out->size);
+			if (!out->buf) {
+				result = ERROR(memory_allocation);
+				goto error_lock;
+			}
+			out->allocated = out->size;
+		}
+
+		result =
+		    LZ4F_decompress(w->dctx, out->buf, &out->size,
+				    in->buf, &in->size, 0);
+
+		if (LZ4F_isError(result)) {
+			lz4mt_errcode = result;
+			result = ERROR(compression_library);
+			goto error_lock;
+		}
+
+		if (result != 0) {
+			result = ERROR(frame_decompress);
+			goto error_lock;
+		}
+
+		/* write result */
+		pthread_mutex_lock(&ctx->write_mutex);
+		result = pt_write(ctx, wl);
+		if (LZ4MT_isError(result))
+			goto error_unlock;
+		pthread_mutex_unlock(&ctx->write_mutex);
+	}
+
+	/* everything is okay */
+	pthread_mutex_lock(&ctx->write_mutex);
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	if (in->allocated)
+		free(in->buf);
+	return 0;
+
+ error_lock:
+	pthread_mutex_lock(&ctx->write_mutex);
+ error_unlock:
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	if (in->allocated)
+		free(in->buf);
+	return (void *)result;
+}
+
+/* single threaded */
+static size_t st_decompress(void *arg)
+{
+	LZ4MT_DCtx *ctx = (LZ4MT_DCtx *) arg;
+	LZ4F_errorCode_t nextToLoad = 0;
+	cwork_t *w = &ctx->cwork[0];
+	LZ4MT_Buffer Out;
+	LZ4MT_Buffer *out = &Out;
+	LZ4MT_Buffer *in = &w->in;
+	void *magic = in->buf;
+	size_t pos = 0;
+	int rv;
+
+	/* allocate space for input buffer */
+	in->size = ctx->inputsize;
+	in->buf = malloc(in->size);
+	if (!in->buf)
+		return ERROR(memory_allocation);
+
+	/* allocate space for output buffer */
+	out->size = ctx->inputsize;
+	out->buf = malloc(out->size);
+	if (!out->buf) {
+		free(in->buf);
+		return ERROR(memory_allocation);
+	}
+
+	/* we have read already 4 bytes */
+	in->size = 4;
+	memcpy(in->buf, magic, in->size);
+
+	nextToLoad =
+	    LZ4F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
+	if (LZ4F_isError(nextToLoad)) {
+		free(in->buf);
+		free(out->buf);
+		return ERROR(compression_library);
+	}
+
+	for (; nextToLoad; pos = 0) {
+		if (nextToLoad > ctx->inputsize)
+			nextToLoad = ctx->inputsize;
+
+		/* read new input */
+		in->size = nextToLoad;
+		rv = ctx->fn_read(ctx->arg_read, in);
+		if (rv == -1) {
+			free(in->buf);
+			free(out->buf);
+			return ERROR(read_fail);
+		}
+
+		/* done, eof reached */
+		if (in->size == 0)
+			break;
+
+		/* still to read, or still to flush */
+		while ((pos < in->size) || (out->size == ctx->inputsize)) {
+			size_t remaining = in->size - pos;
+			out->size = ctx->inputsize;
+
+			/* decompress */
+			nextToLoad =
+			    LZ4F_decompress(w->dctx, out->buf, &out->size,
+					    (unsigned char *)in->buf + pos,
+					    &remaining, NULL);
+			if (LZ4F_isError(nextToLoad)) {
+				free(in->buf);
+				free(out->buf);
+				return ERROR(compression_library);
+			}
+
+			/* have some output */
+			if (out->size) {
+				rv = ctx->fn_write(ctx->arg_write, out);
+				if (rv == -1) {
+					free(in->buf);
+					free(out->buf);
+					return ERROR(write_fail);
+				}
+			}
+
+			if (nextToLoad == 0)
+				break;
+
+			pos += remaining;
+		}
+	}
+
+	/* no error */
+	free(out->buf);
+	free(in->buf);
+	return 0;
+}
+
+size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr)
+{
+	unsigned char buf[4];
+	int t, rv;
+	cwork_t *w = &ctx->cwork[0];
+	LZ4MT_Buffer *in = &w->in;
+
+	if (!ctx)
+		return ERROR(compressionParameter_unsupported);
+
+	/* init reading and writing functions */
+	ctx->fn_read = rdwr->fn_read;
+	ctx->fn_write = rdwr->fn_write;
+	ctx->arg_read = rdwr->arg_read;
+	ctx->arg_write = rdwr->arg_write;
+
+	/* check for LZ4FMT_MAGIC_SKIPPABLE */
+	in->buf = buf;
+	in->size = 4;
+	rv = ctx->fn_read(ctx->arg_read, in);
+	if (rv == -1)
+		return ERROR(read_fail);
+	if (in->size != 4)
+		return ERROR(data_error);
+
+	/* single threaded with unknown sizes */
+	if (MEM_readLE32(buf) != LZ4FMT_MAGIC_SKIPPABLE) {
+
+		/* look for correct magic */
+		if (MEM_readLE32(buf) != LZ4FMT_MAGICNUMBER)
+			return ERROR(data_error);
+
+		/* decompress single threaded */
+		return st_decompress(ctx);
+	}
+
+	/* mark unused */
+	in->buf = 0;
+	in->size = 0;
+	in->allocated = 0;
+
+	/* single threaded, but with known sizes */
+	if (ctx->threads == 1) {
+		/* no pthread_create() needed! */
+		void *p = pt_decompress(w);
+		if (p)
+			return (size_t) p;
+		goto okay;
+	}
+
+	/* multi threaded */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->in.buf = 0;
+		w->in.size = 0;
+		w->in.allocated = 0;
+		pthread_create(&w->pthread, NULL, pt_decompress, w);
+	}
+
+	/* wait for all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		void *p;
+		pthread_join(w->pthread, &p);
+		if (p)
+			return (size_t) p;
+	}
+
+ okay:
+	/* clean up the buffers */
+	while (!list_empty(&ctx->writelist_free)) {
+		struct writelist *wl;
+		struct list_head *entry;
+		entry = list_first(&ctx->writelist_free);
+		wl = list_entry(entry, struct writelist, node);
+		free(wl->out.buf);
+		list_del(&wl->node);
+		free(wl);
+	}
+
+	return 0;
+}
+
+/* returns current uncompressed data size */
+size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->insize;
+}
+
+/* returns the current compressed data size */
+size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->outsize;
+}
+
+/* returns the current compressed frames */
+size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->curframe;
+}
+
+void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx)
+{
+	int t;
+
+	if (!ctx)
+		return;
+
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		LZ4F_freeDecompressionContext(w->dctx);
+	}
+
+	pthread_mutex_destroy(&ctx->read_mutex);
+	pthread_mutex_destroy(&ctx->write_mutex);
+	free(ctx->cwork);
+	free(ctx);
+	ctx = 0;
+
+	return;
+}

C/zstdmt/lz5mt.h

@@ -0,0 +1,162 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/* ***************************************
+ * Defines
+ ****************************************/
+
+#ifndef LZ5MT_H
+#define LZ5MT_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <stddef.h>   /* size_t */
+
+/* current maximum the library will accept */
+#define LZ5MT_THREAD_MAX 128
+#define LZ5MT_LEVEL_MAX   16
+
+#define LZ5FMT_MAGICNUMBER     0x184D2205U
+#define LZ5FMT_MAGIC_SKIPPABLE 0x184D2A50U
+
+/* **************************************
+ * Error Handling
+ ****************************************/
+
+extern size_t lz5mt_errcode;
+
+typedef enum {
+  LZ5MT_error_no_error,
+  LZ5MT_error_memory_allocation,
+  LZ5MT_error_read_fail,
+  LZ5MT_error_write_fail,
+  LZ5MT_error_data_error,
+  LZ5MT_error_frame_compress,
+  LZ5MT_error_frame_decompress,
+  LZ5MT_error_compressionParameter_unsupported,
+  LZ5MT_error_compression_library,
+  LZ5MT_error_maxCode
+} LZ5MT_ErrorCode;
+
+#ifdef ERROR
+#  undef ERROR   /* reported already defined on VS 2015 (Rich Geldreich) */
+#endif
+#define PREFIX(name) LZ5MT_error_##name
+#define ERROR(name) ((size_t)-PREFIX(name))
+extern unsigned LZ5MT_isError(size_t code);
+extern const char* LZ5MT_getErrorString(size_t code);
+
+/* **************************************
+ * Structures
+ ****************************************/
+
+typedef struct {
+	void *buf;		/* ptr to data */
+	size_t size;		/* current filled in buf */
+	size_t allocated;	/* length of buf */
+} LZ5MT_Buffer;
+
+/**
+ * reading and writing functions
+ * - you can use stdio functions or plain read/write
+ * - just write some wrapper on your own
+ * - a sample is given in 7-Zip ZS
+ */
+typedef int (fn_read) (void *args, LZ5MT_Buffer * in);
+typedef int (fn_write) (void *args, LZ5MT_Buffer * out);
+
+typedef struct {
+	fn_read *fn_read;
+	void *arg_read;
+	fn_write *fn_write;
+	void *arg_write;
+} LZ5MT_RdWr_t;
+
+/* **************************************
+ * Compression
+ ****************************************/
+
+typedef struct LZ5MT_CCtx_s LZ5MT_CCtx;
+
+/**
+ * 1) allocate new cctx
+ * - return cctx or zero on error
+ *
+ * @level   - 1 .. 16
+ * @threads - 1 .. LZ5MT_THREAD_MAX
+ * @inputsize - if zero, becomes some optimal value for the level
+ *            - if nonzero, the given value is taken
+ */
+LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize);
+
+/**
+ * 2) threaded compression
+ * - return -1 on error
+ */
+size_t LZ5MT_CompressCCtx(LZ5MT_CCtx * ctx, LZ5MT_RdWr_t * rdwr);
+
+/**
+ * 3) get some statistic
+ */
+size_t LZ5MT_GetFramesCCtx(LZ5MT_CCtx * ctx);
+size_t LZ5MT_GetInsizeCCtx(LZ5MT_CCtx * ctx);
+size_t LZ5MT_GetOutsizeCCtx(LZ5MT_CCtx * ctx);
+
+/**
+ * 4) free cctx
+ * - no special return value
+ */
+void LZ5MT_freeCCtx(LZ5MT_CCtx * ctx);
+
+/* **************************************
+ * Decompression - TODO, but it's easy...
+ ****************************************/
+
+typedef struct LZ5MT_DCtx_s LZ5MT_DCtx;
+
+/**
+ * 1) allocate new cctx
+ * - return cctx or zero on error
+ *
+ * @level   - 1 .. 22
+ * @threads - 1 .. LZ5MT_THREAD_MAX
+ * @srclen  - the max size of src for LZ5MT_CompressCCtx()
+ * @dstlen  - the min size of dst
+ */
+LZ5MT_DCtx *LZ5MT_createDCtx(int threads, int inputsize);
+
+/**
+ * 2) threaded compression
+ * - return -1 on error
+ */
+size_t LZ5MT_DecompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr);
+
+/**
+ * 3) get some statistic
+ */
+size_t LZ5MT_GetFramesDCtx(LZ5MT_DCtx * ctx);
+size_t LZ5MT_GetInsizeDCtx(LZ5MT_DCtx * ctx);
+size_t LZ5MT_GetOutsizeDCtx(LZ5MT_DCtx * ctx);
+
+/**
+ * 4) free cctx
+ * - no special return value
+ */
+void LZ5MT_freeDCtx(LZ5MT_DCtx * ctx);
+
+#if defined (__cplusplus)
+}
+#endif
+#endif				/* LZ5MT_H */

C/zstdmt/lz5mt_common.c

@@ -0,0 +1,62 @@
+
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include "lz5frame.h"
+#include "lz5mt.h"
+
+/* will be used for lib errors */
+size_t lz5mt_errcode;
+
+/* ****************************************
+ * LZ5MT Error Management
+ ******************************************/
+
+/**
+ * LZ5MT_isError() - tells if a return value is an error code
+ */
+unsigned LZ5MT_isError(size_t code)
+{
+	return (code > ERROR(maxCode));
+}
+
+/**
+ * LZ5MT_getErrorString() - give error code string from function result
+ */
+const char *LZ5MT_getErrorString(size_t code)
+{
+	if (LZ5F_isError(lz5mt_errcode))
+		return LZ5F_getErrorName(lz5mt_errcode);
+
+	static const char *notErrorCode = "Unspecified error lz4mt code";
+	switch ((LZ5MT_ErrorCode)(0-code)) {
+	case PREFIX(no_error):
+		return "No error detected";
+	case PREFIX(memory_allocation):
+		return "Allocation error : not enough memory";
+	case PREFIX(read_fail):
+		return "Read failure";
+	case PREFIX(write_fail):
+		return "Write failure";
+	case PREFIX(data_error):
+		return "Malformed input";
+	case PREFIX(frame_compress):
+		return "Could not compress frame at once";
+	case PREFIX(frame_decompress):
+		return "Could not decompress frame at once";
+	case PREFIX(compressionParameter_unsupported):
+		return "Compression parameter is out of bound";
+	case PREFIX(compression_library):
+		return "Compression library reports failure";
+	case PREFIX(maxCode):
+	default:
+		return notErrorCode;
+	}
+}

C/zstdmt/lz5mt_compress.c

@@ -0,0 +1,377 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#define LZ5F_DISABLE_OBSOLETE_ENUMS
+#include "lz5frame.h"
+
+#include "mem.h"
+#include "threading.h"
+#include "list.h"
+#include "lz5mt.h"
+
+/**
+ * multi threaded lz5 - multiple workers version
+ *
+ * - each thread works on his own
+ * - no main thread which does reading and then starting the work
+ * - needs a callback for reading / writing
+ * - each worker does his:
+ *   1) get read mutex and read some input
+ *   2) release read mutex and do compression
+ *   3) get write mutex and write result
+ *   4) begin with step 1 again, until no input
+ */
+
+/* worker for compression */
+typedef struct {
+	LZ5MT_CCtx *ctx;
+	LZ5F_preferences_t zpref;
+	pthread_t pthread;
+} cwork_t;
+
+struct writelist;
+struct writelist {
+	size_t frame;
+	LZ5MT_Buffer out;
+	struct list_head node;
+};
+
+struct LZ5MT_CCtx_s {
+
+	/* level: 1..22 */
+	int level;
+
+	/* threads: 1..LZ5MT_THREAD_MAX */
+	int threads;
+
+	/* should be used for read from input */
+	int inputsize;
+
+	/* statistic */
+	size_t insize;
+	size_t outsize;
+	size_t curframe;
+	size_t frames;
+
+	/* threading */
+	cwork_t *cwork;
+
+	/* reading input */
+	pthread_mutex_t read_mutex;
+	fn_read *fn_read;
+	void *arg_read;
+
+	/* writing output */
+	pthread_mutex_t write_mutex;
+	fn_write *fn_write;
+	void *arg_write;
+
+	/* lists for writing queue */
+	struct list_head writelist_free;
+	struct list_head writelist_busy;
+	struct list_head writelist_done;
+};
+
+/* **************************************
+ * Compression
+ ****************************************/
+
+LZ5MT_CCtx *LZ5MT_createCCtx(int threads, int level, int inputsize)
+{
+	LZ5MT_CCtx *ctx;
+	int t;
+
+	/* allocate ctx */
+	ctx = (LZ5MT_CCtx *) malloc(sizeof(LZ5MT_CCtx));
+	if (!ctx)
+		return 0;
+
+	/* check threads value */
+	if (threads < 1 || threads > LZ5MT_THREAD_MAX)
+		return 0;
+
+	/* check level */
+	if (level < 1 || level > LZ5MT_LEVEL_MAX)
+		return 0;
+
+	/* calculate chunksize for one thread */
+	if (inputsize)
+		ctx->inputsize = inputsize;
+	else
+		ctx->inputsize = 1024 * 64;
+
+	/* setup ctx */
+	ctx->level = level;
+	ctx->threads = threads;
+	ctx->insize = 0;
+	ctx->outsize = 0;
+	ctx->frames = 0;
+	ctx->curframe = 0;
+
+	pthread_mutex_init(&ctx->read_mutex, NULL);
+	pthread_mutex_init(&ctx->write_mutex, NULL);
+
+	/* free -> busy -> out -> free -> ... */
+	INIT_LIST_HEAD(&ctx->writelist_free);	/* free, can be used */
+	INIT_LIST_HEAD(&ctx->writelist_busy);	/* busy */
+	INIT_LIST_HEAD(&ctx->writelist_done);	/* can be written */
+
+	ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
+	if (!ctx->cwork)
+		goto err_cwork;
+
+	for (t = 0; t < threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->ctx = ctx;
+
+		/* setup preferences for that thread */
+		memset(&w->zpref, 0, sizeof(LZ5F_preferences_t));
+		w->zpref.compressionLevel = level;
+		w->zpref.frameInfo.blockMode = LZ5F_blockLinked;
+		w->zpref.frameInfo.contentSize = 1;
+		w->zpref.frameInfo.contentChecksumFlag =
+		    LZ5F_contentChecksumEnabled;
+
+	}
+
+	return ctx;
+
+ err_cwork:
+	free(ctx);
+
+	return 0;
+}
+
+/**
+ * pt_write - queue for compressed output
+ */
+static size_t pt_write(LZ5MT_CCtx * ctx, struct writelist *wl)
+{
+	struct list_head *entry;
+
+	/* move the entry to the done list */
+	list_move(&wl->node, &ctx->writelist_done);
+
+	/* the entry isn't the currently needed, return...  */
+	if (wl->frame != ctx->curframe)
+		return 0;
+
+ again:
+	/* check, what can be written ... */
+	list_for_each(entry, &ctx->writelist_done) {
+		wl = list_entry(entry, struct writelist, node);
+		if (wl->frame == ctx->curframe) {
+			int rv = ctx->fn_write(ctx->arg_write, &wl->out);
+			if (rv == -1)
+				return ERROR(write_fail);
+			ctx->outsize += wl->out.size;
+			ctx->curframe++;
+			list_move(entry, &ctx->writelist_free);
+			goto again;
+		}
+	}
+
+	return 0;
+}
+
+static void *pt_compress(void *arg)
+{
+	cwork_t *w = (cwork_t *) arg;
+	LZ5MT_CCtx *ctx = w->ctx;
+	size_t result;
+	LZ5MT_Buffer in;
+
+	/* inbuf is constant */
+	in.size = ctx->inputsize;
+	in.buf = malloc(in.size);
+	if (!in.buf)
+		return (void *)ERROR(memory_allocation);
+
+	for (;;) {
+		struct list_head *entry;
+		struct writelist *wl;
+		int rv;
+
+		/* allocate space for new output */
+		pthread_mutex_lock(&ctx->write_mutex);
+		if (!list_empty(&ctx->writelist_free)) {
+			/* take unused entry */
+			entry = list_first(&ctx->writelist_free);
+			wl = list_entry(entry, struct writelist, node);
+			wl->out.size =
+			    LZ5F_compressFrameBound(ctx->inputsize,
+						    &w->zpref) + 12;
+			list_move(entry, &ctx->writelist_busy);
+		} else {
+			/* allocate new one */
+			wl = (struct writelist *)
+			    malloc(sizeof(struct writelist));
+			if (!wl) {
+				pthread_mutex_unlock(&ctx->write_mutex);
+				return (void *)ERROR(memory_allocation);
+			}
+			wl->out.size =
+			    LZ5F_compressFrameBound(ctx->inputsize,
+						    &w->zpref) + 12;;
+			wl->out.buf = malloc(wl->out.size);
+			if (!wl->out.buf) {
+				pthread_mutex_unlock(&ctx->write_mutex);
+				return (void *)ERROR(memory_allocation);
+			}
+			list_add(&wl->node, &ctx->writelist_busy);
+		}
+		pthread_mutex_unlock(&ctx->write_mutex);
+
+		/* read new input */
+		pthread_mutex_lock(&ctx->read_mutex);
+		in.size = ctx->inputsize;
+		rv = ctx->fn_read(ctx->arg_read, &in);
+		if (rv == -1) {
+			pthread_mutex_unlock(&ctx->read_mutex);
+			return (void *)ERROR(read_fail);
+		}
+
+		/* eof */
+		if (in.size == 0) {
+			free(in.buf);
+			pthread_mutex_unlock(&ctx->read_mutex);
+
+			pthread_mutex_lock(&ctx->write_mutex);
+			list_move(&wl->node, &ctx->writelist_free);
+			pthread_mutex_unlock(&ctx->write_mutex);
+
+			goto okay;
+		}
+		ctx->insize += in.size;
+		wl->frame = ctx->frames++;
+		pthread_mutex_unlock(&ctx->read_mutex);
+
+		/* compress whole frame */
+		result =
+		    LZ5F_compressFrame((unsigned char *)wl->out.buf + 12,
+				       wl->out.size - 12, in.buf, in.size,
+				       &w->zpref);
+		if (LZ5F_isError(result)) {
+			pthread_mutex_lock(&ctx->write_mutex);
+			list_move(&wl->node, &ctx->writelist_free);
+			pthread_mutex_unlock(&ctx->write_mutex);
+			/* user can lookup that code */
+			lz5mt_errcode = result;
+			return (void *)ERROR(compression_library);
+		}
+
+		/* write skippable frame */
+		MEM_writeLE32((unsigned char *)wl->out.buf + 0,
+			      LZ5FMT_MAGIC_SKIPPABLE);
+		MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4);
+		MEM_writeLE32((unsigned char *)wl->out.buf + 8,
+			      (U32)result);
+		wl->out.size = result + 12;
+
+		/* write result */
+		pthread_mutex_lock(&ctx->write_mutex);
+		result = pt_write(ctx, wl);
+		pthread_mutex_unlock(&ctx->write_mutex);
+		if (LZ5MT_isError(result))
+			return (void *)result;
+	}
+
+ okay:
+	return 0;
+}
+
+size_t LZ5MT_CompressCCtx(LZ5MT_CCtx * ctx, LZ5MT_RdWr_t * rdwr)
+{
+	int t;
+
+	if (!ctx)
+		return ERROR(compressionParameter_unsupported);
+
+	/* init reading and writing functions */
+	ctx->fn_read = rdwr->fn_read;
+	ctx->fn_write = rdwr->fn_write;
+	ctx->arg_read = rdwr->arg_read;
+	ctx->arg_write = rdwr->arg_write;
+
+	/* start all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		pthread_create(&w->pthread, NULL, pt_compress, w);
+	}
+
+	/* wait for all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		void *p;
+		pthread_join(w->pthread, &p);
+		if (p)
+			return (size_t) p;
+	}
+
+	/* clean up lists */
+	while (!list_empty(&ctx->writelist_free)) {
+		struct writelist *wl;
+		struct list_head *entry;
+		entry = list_first(&ctx->writelist_free);
+		wl = list_entry(entry, struct writelist, node);
+		free(wl->out.buf);
+		list_del(&wl->node);
+		free(wl);
+	}
+
+	return 0;
+}
+
+/* returns current uncompressed data size */
+size_t LZ5MT_GetInsizeCCtx(LZ5MT_CCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->insize;
+}
+
+/* returns the current compressed data size */
+size_t LZ5MT_GetOutsizeCCtx(LZ5MT_CCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->outsize;
+}
+
+/* returns the current compressed frames */
+size_t LZ5MT_GetFramesCCtx(LZ5MT_CCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->curframe;
+}
+
+void LZ5MT_freeCCtx(LZ5MT_CCtx * ctx)
+{
+	if (!ctx)
+		return;
+
+	pthread_mutex_destroy(&ctx->read_mutex);
+	pthread_mutex_destroy(&ctx->write_mutex);
+	free(ctx->cwork);
+	free(ctx);
+	ctx = 0;
+
+	return;
+}

C/zstdmt/lz5mt_decompress.c

@@ -0,0 +1,588 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#define LZ5F_DISABLE_OBSOLETE_ENUMS
+#include "lz5frame.h"
+
+#include "mem.h"
+#include "threading.h"
+#include "list.h"
+#include "lz5mt.h"
+
+/**
+ * multi threaded lz5 - multiple workers version
+ *
+ * - each thread works on his own
+ * - no main thread which does reading and then starting the work
+ * - needs a callback for reading / writing
+ * - each worker does his:
+ *   1) get read mutex and read some input
+ *   2) release read mutex and do compression
+ *   3) get write mutex and write result
+ *   4) begin with step 1 again, until no input
+ */
+
+/* worker for compression */
+typedef struct {
+	LZ5MT_DCtx *ctx;
+	pthread_t pthread;
+	LZ5MT_Buffer in;
+	LZ5F_decompressionContext_t dctx;
+} cwork_t;
+
+struct writelist;
+struct writelist {
+	size_t frame;
+	LZ5MT_Buffer out;
+	struct list_head node;
+};
+
+struct LZ5MT_DCtx_s {
+
+	/* threads: 1..LZ5MT_THREAD_MAX */
+	int threads;
+
+	/* should be used for read from input */
+	size_t inputsize;
+
+	/* statistic */
+	size_t insize;
+	size_t outsize;
+	size_t curframe;
+	size_t frames;
+
+	/* threading */
+	cwork_t *cwork;
+
+	/* reading input */
+	pthread_mutex_t read_mutex;
+	fn_read *fn_read;
+	void *arg_read;
+
+	/* writing output */
+	pthread_mutex_t write_mutex;
+	fn_write *fn_write;
+	void *arg_write;
+
+	/* lists for writing queue */
+	struct list_head writelist_free;
+	struct list_head writelist_busy;
+	struct list_head writelist_done;
+};
+
+/* **************************************
+ * Decompression
+ ****************************************/
+
+LZ5MT_DCtx *LZ5MT_createDCtx(int threads, int inputsize)
+{
+	LZ5MT_DCtx *ctx;
+	int t;
+
+	/* allocate ctx */
+	ctx = (LZ5MT_DCtx *) malloc(sizeof(LZ5MT_DCtx));
+	if (!ctx)
+		return 0;
+
+	/* check threads value */
+	if (threads < 1 || threads > LZ5MT_THREAD_MAX)
+		return 0;
+
+	/* setup ctx */
+	ctx->threads = threads;
+	ctx->insize = 0;
+	ctx->outsize = 0;
+	ctx->frames = 0;
+	ctx->curframe = 0;
+
+	/* will be used for single stream only */
+	if (inputsize)
+		ctx->inputsize = inputsize;
+	else
+		ctx->inputsize = 1024 * 64;	/* 64K buffer */
+
+	pthread_mutex_init(&ctx->read_mutex, NULL);
+	pthread_mutex_init(&ctx->write_mutex, NULL);
+
+	INIT_LIST_HEAD(&ctx->writelist_free);
+	INIT_LIST_HEAD(&ctx->writelist_busy);
+	INIT_LIST_HEAD(&ctx->writelist_done);
+
+	ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
+	if (!ctx->cwork)
+		goto err_cwork;
+
+	for (t = 0; t < threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->ctx = ctx;
+
+		/* setup thread work */
+		LZ5F_createDecompressionContext(&w->dctx, LZ5F_VERSION);
+	}
+
+	return ctx;
+
+ err_cwork:
+	free(ctx);
+
+	return 0;
+}
+
+/**
+ * pt_write - queue for decompressed output
+ */
+static size_t pt_write(LZ5MT_DCtx * ctx, struct writelist *wl)
+{
+	struct list_head *entry;
+
+	/* move the entry to the done list */
+	list_move(&wl->node, &ctx->writelist_done);
+ again:
+	/* check, what can be written ... */
+	list_for_each(entry, &ctx->writelist_done) {
+		wl = list_entry(entry, struct writelist, node);
+		if (wl->frame == ctx->curframe) {
+			int rv = ctx->fn_write(ctx->arg_write, &wl->out);
+			if (rv == -1)
+				return ERROR(write_fail);
+			ctx->outsize += wl->out.size;
+			ctx->curframe++;
+			list_move(entry, &ctx->writelist_free);
+			goto again;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * pt_read - read compressed output
+ */
+static size_t pt_read(LZ5MT_DCtx * ctx, LZ5MT_Buffer * in, size_t * frame)
+{
+	unsigned char hdrbuf[12];
+	LZ5MT_Buffer hdr;
+	int rv;
+
+	/* read skippable frame (8 or 12 bytes) */
+	pthread_mutex_lock(&ctx->read_mutex);
+
+	/* special case, first 4 bytes already read */
+	if (ctx->frames == 0) {
+		hdr.buf = hdrbuf + 4;
+		hdr.size = 8;
+		rv = ctx->fn_read(ctx->arg_read, &hdr);
+		if (rv == -1)
+			goto error_read;
+		if (hdr.size != 8)
+			goto error_read;
+		hdr.buf = hdrbuf;
+	} else {
+		hdr.buf = hdrbuf;
+		hdr.size = 12;
+		rv = ctx->fn_read(ctx->arg_read, &hdr);
+		if (rv == -1)
+			goto error_read;
+		/* eof reached ? */
+		if (hdr.size == 0) {
+			pthread_mutex_unlock(&ctx->read_mutex);
+			in->size = 0;
+			return 0;
+		}
+		if (hdr.size != 12)
+			goto error_read;
+		if (MEM_readLE32((unsigned char *)hdr.buf + 0) !=
+		    LZ5FMT_MAGIC_SKIPPABLE)
+			goto error_data;
+	}
+
+	/* check header data */
+	if (MEM_readLE32((unsigned char *)hdr.buf + 4) != 4)
+		goto error_data;
+
+	ctx->insize += 12;
+	/* read new inputsize */
+	{
+		size_t toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
+		if (in->allocated < toRead) {
+			/* need bigger input buffer */
+			if (in->allocated)
+				in->buf = realloc(in->buf, toRead);
+			else
+				in->buf = malloc(toRead);
+			if (!in->buf)
+				goto error_nomem;
+			in->allocated = toRead;
+		}
+
+		in->size = toRead;
+		rv = ctx->fn_read(ctx->arg_read, in);
+		/* generic read failure! */
+		if (rv == -1)
+			goto error_read;
+		/* needed more bytes! */
+		if (in->size != toRead)
+			goto error_data;
+
+		ctx->insize += in->size;
+	}
+	*frame = ctx->frames++;
+	pthread_mutex_unlock(&ctx->read_mutex);
+
+	/* done, no error */
+	return 0;
+
+ error_data:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ERROR(data_error);
+ error_read:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ERROR(read_fail);
+ error_nomem:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ERROR(memory_allocation);
+}
+
+static void *pt_decompress(void *arg)
+{
+	cwork_t *w = (cwork_t *) arg;
+	LZ5MT_Buffer *in = &w->in;
+	LZ5MT_DCtx *ctx = w->ctx;
+	size_t result = 0;
+	struct writelist *wl;
+
+	for (;;) {
+		struct list_head *entry;
+		LZ5MT_Buffer *out;
+
+		/* allocate space for new output */
+		pthread_mutex_lock(&ctx->write_mutex);
+		if (!list_empty(&ctx->writelist_free)) {
+			/* take unused entry */
+			entry = list_first(&ctx->writelist_free);
+			wl = list_entry(entry, struct writelist, node);
+			list_move(entry, &ctx->writelist_busy);
+		} else {
+			/* allocate new one */
+			wl = (struct writelist *)
+			    malloc(sizeof(struct writelist));
+			if (!wl) {
+				result = ERROR(memory_allocation);
+				goto error_unlock;
+			}
+			wl->out.buf = 0;
+			wl->out.size = 0;
+			wl->out.allocated = 0;
+			list_add(&wl->node, &ctx->writelist_busy);
+		}
+		pthread_mutex_unlock(&ctx->write_mutex);
+		out = &wl->out;
+
+		/* zero should not happen here! */
+		result = pt_read(ctx, in, &wl->frame);
+		if (in->size == 0)
+			break;
+
+		if (LZ5MT_isError(result)) {
+			list_move(&wl->node, &ctx->writelist_free);
+
+			goto error_lock;
+		}
+
+		{
+			/* get frame size for output buffer */
+			unsigned char *src = (unsigned char *)in->buf + 6;
+			out->size = (size_t) MEM_readLE64(src);
+		}
+
+		if (out->allocated < out->size) {
+			if (out->allocated)
+				out->buf = realloc(out->buf, out->size);
+			else
+				out->buf = malloc(out->size);
+			if (!out->buf) {
+				result = ERROR(memory_allocation);
+				goto error_lock;
+			}
+			out->allocated = out->size;
+		}
+
+		result =
+		    LZ5F_decompress(w->dctx, out->buf, &out->size,
+				    in->buf, &in->size, 0);
+
+		if (LZ5F_isError(result)) {
+			lz5mt_errcode = result;
+			result = ERROR(compression_library);
+			goto error_lock;
+		}
+
+		if (result != 0) {
+			result = ERROR(frame_decompress);
+			goto error_lock;
+		}
+
+		/* write result */
+		pthread_mutex_lock(&ctx->write_mutex);
+		result = pt_write(ctx, wl);
+		if (LZ5MT_isError(result))
+			goto error_unlock;
+		pthread_mutex_unlock(&ctx->write_mutex);
+	}
+
+	/* everything is okay */
+	pthread_mutex_lock(&ctx->write_mutex);
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	if (in->allocated)
+		free(in->buf);
+	return 0;
+
+ error_lock:
+	pthread_mutex_lock(&ctx->write_mutex);
+ error_unlock:
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	if (in->allocated)
+		free(in->buf);
+	return (void *)result;
+}
+
+/* single threaded */
+static size_t st_decompress(void *arg)
+{
+	LZ5MT_DCtx *ctx = (LZ5MT_DCtx *) arg;
+	LZ5F_errorCode_t nextToLoad = 0;
+	cwork_t *w = &ctx->cwork[0];
+	LZ5MT_Buffer Out;
+	LZ5MT_Buffer *out = &Out;
+	LZ5MT_Buffer *in = &w->in;
+	void *magic = in->buf;
+	size_t pos = 0;
+	int rv;
+
+	/* allocate space for input buffer */
+	in->size = ctx->inputsize;
+	in->buf = malloc(in->size);
+	if (!in->buf)
+		return ERROR(memory_allocation);
+
+	/* allocate space for output buffer */
+	out->size = ctx->inputsize;
+	out->buf = malloc(out->size);
+	if (!out->buf) {
+		free(in->buf);
+		return ERROR(memory_allocation);
+	}
+
+	/* we have read already 4 bytes */
+	in->size = 4;
+	memcpy(in->buf, magic, in->size);
+
+	nextToLoad =
+	    LZ5F_decompress(w->dctx, out->buf, &pos, in->buf, &in->size, 0);
+	if (LZ5F_isError(nextToLoad)) {
+		free(in->buf);
+		free(out->buf);
+		return ERROR(compression_library);
+	}
+
+	for (; nextToLoad; pos = 0) {
+		if (nextToLoad > ctx->inputsize)
+			nextToLoad = ctx->inputsize;
+
+		/* read new input */
+		in->size = nextToLoad;
+		rv = ctx->fn_read(ctx->arg_read, in);
+		if (rv == -1) {
+			free(in->buf);
+			free(out->buf);
+			return ERROR(read_fail);
+		}
+
+		/* done, eof reached */
+		if (in->size == 0)
+			break;
+
+		/* still to read, or still to flush */
+		while ((pos < in->size) || (out->size == ctx->inputsize)) {
+			size_t remaining = in->size - pos;
+			out->size = ctx->inputsize;
+
+			/* decompress */
+			nextToLoad =
+			    LZ5F_decompress(w->dctx, out->buf, &out->size,
+					    (unsigned char *)in->buf + pos,
+					    &remaining, NULL);
+			if (LZ5F_isError(nextToLoad)) {
+				free(in->buf);
+				free(out->buf);
+				return ERROR(compression_library);
+			}
+
+			/* have some output */
+			if (out->size) {
+				rv = ctx->fn_write(ctx->arg_write, out);
+				if (rv == -1) {
+					free(in->buf);
+					free(out->buf);
+					return ERROR(write_fail);
+				}
+			}
+
+			if (nextToLoad == 0)
+				break;
+
+			pos += remaining;
+		}
+	}
+
+	/* no error */
+	free(out->buf);
+	free(in->buf);
+	return 0;
+}
+
+size_t LZ5MT_DecompressDCtx(LZ5MT_DCtx * ctx, LZ5MT_RdWr_t * rdwr)
+{
+	unsigned char buf[4];
+	int t, rv;
+	cwork_t *w = &ctx->cwork[0];
+	LZ5MT_Buffer *in = &w->in;
+
+	if (!ctx)
+		return ERROR(compressionParameter_unsupported);
+
+	/* init reading and writing functions */
+	ctx->fn_read = rdwr->fn_read;
+	ctx->fn_write = rdwr->fn_write;
+	ctx->arg_read = rdwr->arg_read;
+	ctx->arg_write = rdwr->arg_write;
+
+	/* check for LZ5FMT_MAGIC_SKIPPABLE */
+	in->buf = buf;
+	in->size = 4;
+	rv = ctx->fn_read(ctx->arg_read, in);
+	if (rv == -1)
+		return ERROR(read_fail);
+	if (in->size != 4)
+		return ERROR(data_error);
+
+	/* single threaded with unknown sizes */
+	if (MEM_readLE32(buf) != LZ5FMT_MAGIC_SKIPPABLE) {
+
+		/* look for correct magic */
+		if (MEM_readLE32(buf) != LZ5FMT_MAGICNUMBER)
+			return ERROR(data_error);
+
+		/* decompress single threaded */
+		return st_decompress(ctx);
+	}
+
+	/* mark unused */
+	in->buf = 0;
+	in->size = 0;
+	in->allocated = 0;
+
+	/* single threaded, but with known sizes */
+	if (ctx->threads == 1) {
+		/* no pthread_create() needed! */
+		void *p = pt_decompress(w);
+		if (p)
+			return (size_t) p;
+		goto okay;
+	}
+
+	/* multi threaded */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->in.buf = 0;
+		w->in.size = 0;
+		w->in.allocated = 0;
+		pthread_create(&w->pthread, NULL, pt_decompress, w);
+	}
+
+	/* wait for all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		void *p;
+		pthread_join(w->pthread, &p);
+		if (p)
+			return (size_t) p;
+	}
+
+ okay:
+	/* clean up the buffers */
+	while (!list_empty(&ctx->writelist_free)) {
+		struct writelist *wl;
+		struct list_head *entry;
+		entry = list_first(&ctx->writelist_free);
+		wl = list_entry(entry, struct writelist, node);
+		free(wl->out.buf);
+		list_del(&wl->node);
+		free(wl);
+	}
+
+	return 0;
+}
+
+/* returns current uncompressed data size */
+size_t LZ5MT_GetInsizeDCtx(LZ5MT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->insize;
+}
+
+/* returns the current compressed data size */
+size_t LZ5MT_GetOutsizeDCtx(LZ5MT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->outsize;
+}
+
+/* returns the current compressed frames */
+size_t LZ5MT_GetFramesDCtx(LZ5MT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->curframe;
+}
+
+void LZ5MT_freeDCtx(LZ5MT_DCtx * ctx)
+{
+	int t;
+
+	if (!ctx)
+		return;
+
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		LZ5F_freeDecompressionContext(w->dctx);
+	}
+
+	pthread_mutex_destroy(&ctx->read_mutex);
+	pthread_mutex_destroy(&ctx->write_mutex);
+	free(ctx->cwork);
+	free(ctx);
+	ctx = 0;
+
+	return;
+}

C/zstdmt/mem.h

@@ -0,0 +1,379 @@
+
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef MEM_H_MODULE
+#define MEM_H_MODULE
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+/*-****************************************
+*  Dependencies
+******************************************/
+#include <stddef.h>     /* size_t, ptrdiff_t */
+#include <string.h>     /* memcpy */
+
+
+/*-****************************************
+*  Compiler specifics
+******************************************/
+#if defined(_MSC_VER)   /* Visual Studio */
+#   include <stdlib.h>  /* _byteswap_ulong */
+#   include <intrin.h>  /* _byteswap_* */
+#endif
+#if defined(__GNUC__)
+#  define MEM_STATIC static __inline __attribute__((unused))
+#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
+#  define MEM_STATIC static inline
+#elif defined(_MSC_VER)
+#  define MEM_STATIC static __inline
+#else
+#  define MEM_STATIC static  /* this version may generate warnings for unused static functions; disable the relevant warning */
+#endif
+
+/* code only tested on 32 and 64 bits systems */
+#define MEM_STATIC_ASSERT(c)   { enum { XXH_static_assert = 1/(int)(!!(c)) }; }
+MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
+
+/* define likely() and unlikely() */
+#if defined(__GNUC__) || (__INTEL_COMPILER >= 800) || defined(__clang__)
+#  define expect(expr,value)    (__builtin_expect ((expr),(value)) )
+#else
+#  define expect(expr,value)    (expr)
+#endif
+#define likely(expr)     expect((expr) != 0, 1)
+#define unlikely(expr)   expect((expr) != 0, 0)
+
+/*-**************************************************************
+*  Basic Types
+*****************************************************************/
+#if  !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
+# include <stdint.h>
+  typedef  uint8_t BYTE;
+  typedef uint16_t U16;
+  typedef  int16_t S16;
+  typedef uint32_t U32;
+  typedef  int32_t S32;
+  typedef uint64_t U64;
+  typedef  int64_t S64;
+#else
+  typedef unsigned char       BYTE;
+  typedef unsigned short      U16;
+  typedef   signed short      S16;
+  typedef unsigned int        U32;
+  typedef   signed int        S32;
+  typedef unsigned long long  U64;
+  typedef   signed long long  S64;
+#endif
+
+
+/*-**************************************************************
+*  Memory I/O
+*****************************************************************/
+/* MEM_FORCE_MEMORY_ACCESS :
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
+ * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
+ * The below switch allow to select different access method for improved performance.
+ * Method 0 (default) : use `memcpy()`. Safe and portable.
+ * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable).
+ *            This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
+ * Method 2 : direct access. This method is portable but violate C standard.
+ *            It can generate buggy code on targets depending on alignment.
+ *            In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6)
+ * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
+ * Prefer these methods in priority order (0 > 1 > 2)
+ */
+#ifndef MEM_FORCE_MEMORY_ACCESS   /* can be defined externally, on command line for example */
+#  if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
+#    define MEM_FORCE_MEMORY_ACCESS 2
+#  elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \
+  (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) ))
+#    define MEM_FORCE_MEMORY_ACCESS 1
+#  endif
+#endif
+
+MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
+MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
+
+MEM_STATIC unsigned MEM_isLittleEndian(void)
+{
+    const union { U32 u; BYTE c[4]; } one = { 1 };   /* don't use static : performance detrimental  */
+    return one.c[0];
+}
+
+#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
+
+/* violates C standard, by lying on structure alignment.
+Only use if no other choice to achieve best performance on target platform */
+MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
+MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
+MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
+MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
+
+#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
+
+/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
+/* currently only defined for gcc and icc */
+#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
+	__pragma( pack(push, 1) )
+    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign;
+    __pragma( pack(pop) )
+#else
+    typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign;
+#endif
+
+MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; }
+MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; }
+MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; }
+MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; }
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; }
+MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; }
+MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; }
+
+#else
+
+/* default method, safe and standard.
+   can sometimes prove slower */
+
+MEM_STATIC U16 MEM_read16(const void* memPtr)
+{
+    U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U32 MEM_read32(const void* memPtr)
+{
+    U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC U64 MEM_read64(const void* memPtr)
+{
+    U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC size_t MEM_readST(const void* memPtr)
+{
+    size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
+}
+
+MEM_STATIC void MEM_write16(void* memPtr, U16 value)
+{
+    memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write32(void* memPtr, U32 value)
+{
+    memcpy(memPtr, &value, sizeof(value));
+}
+
+MEM_STATIC void MEM_write64(void* memPtr, U64 value)
+{
+    memcpy(memPtr, &value, sizeof(value));
+}
+
+#endif /* MEM_FORCE_MEMORY_ACCESS */
+
+MEM_STATIC U32 MEM_swap32(U32 in)
+{
+#if defined(_MSC_VER)     /* Visual Studio */
+    return _byteswap_ulong(in);
+#elif defined (__GNUC__)
+    return __builtin_bswap32(in);
+#else
+    return  ((in << 24) & 0xff000000 ) |
+            ((in <<  8) & 0x00ff0000 ) |
+            ((in >>  8) & 0x0000ff00 ) |
+            ((in >> 24) & 0x000000ff );
+#endif
+}
+
+MEM_STATIC U64 MEM_swap64(U64 in)
+{
+#if defined(_MSC_VER)     /* Visual Studio */
+    return _byteswap_uint64(in);
+#elif defined (__GNUC__)
+    return __builtin_bswap64(in);
+#else
+    return  ((in << 56) & 0xff00000000000000ULL) |
+            ((in << 40) & 0x00ff000000000000ULL) |
+            ((in << 24) & 0x0000ff0000000000ULL) |
+            ((in << 8)  & 0x000000ff00000000ULL) |
+            ((in >> 8)  & 0x00000000ff000000ULL) |
+            ((in >> 24) & 0x0000000000ff0000ULL) |
+            ((in >> 40) & 0x000000000000ff00ULL) |
+            ((in >> 56) & 0x00000000000000ffULL);
+#endif
+}
+
+MEM_STATIC size_t MEM_swapST(size_t in)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_swap32((U32)in);
+    else
+        return (size_t)MEM_swap64((U64)in);
+}
+
+/*=== Little endian r/w ===*/
+
+MEM_STATIC U16 MEM_readLE16(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read16(memPtr);
+    else {
+        const BYTE* p = (const BYTE*)memPtr;
+        return (U16)(p[0] + (p[1]<<8));
+    }
+}
+
+MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
+{
+    if (MEM_isLittleEndian()) {
+        MEM_write16(memPtr, val);
+    } else {
+        BYTE* p = (BYTE*)memPtr;
+        p[0] = (BYTE)val;
+        p[1] = (BYTE)(val>>8);
+    }
+}
+
+MEM_STATIC U32 MEM_readLE24(const void* memPtr)
+{
+    return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
+}
+
+MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
+{
+    MEM_writeLE16(memPtr, (U16)val);
+    ((BYTE*)memPtr)[2] = (BYTE)(val>>16);
+}
+
+MEM_STATIC U32 MEM_readLE32(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read32(memPtr);
+    else
+        return MEM_swap32(MEM_read32(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
+{
+    if (MEM_isLittleEndian())
+        MEM_write32(memPtr, val32);
+    else
+        MEM_write32(memPtr, MEM_swap32(val32));
+}
+
+MEM_STATIC U64 MEM_readLE64(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_read64(memPtr);
+    else
+        return MEM_swap64(MEM_read64(memPtr));
+}
+
+MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
+{
+    if (MEM_isLittleEndian())
+        MEM_write64(memPtr, val64);
+    else
+        MEM_write64(memPtr, MEM_swap64(val64));
+}
+
+MEM_STATIC size_t MEM_readLEST(const void* memPtr)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_readLE32(memPtr);
+    else
+        return (size_t)MEM_readLE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
+{
+    if (MEM_32bits())
+        MEM_writeLE32(memPtr, (U32)val);
+    else
+        MEM_writeLE64(memPtr, (U64)val);
+}
+
+/*=== Big endian r/w ===*/
+
+MEM_STATIC U32 MEM_readBE32(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_swap32(MEM_read32(memPtr));
+    else
+        return MEM_read32(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
+{
+    if (MEM_isLittleEndian())
+        MEM_write32(memPtr, MEM_swap32(val32));
+    else
+        MEM_write32(memPtr, val32);
+}
+
+MEM_STATIC U64 MEM_readBE64(const void* memPtr)
+{
+    if (MEM_isLittleEndian())
+        return MEM_swap64(MEM_read64(memPtr));
+    else
+        return MEM_read64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
+{
+    if (MEM_isLittleEndian())
+        MEM_write64(memPtr, MEM_swap64(val64));
+    else
+        MEM_write64(memPtr, val64);
+}
+
+MEM_STATIC size_t MEM_readBEST(const void* memPtr)
+{
+    if (MEM_32bits())
+        return (size_t)MEM_readBE32(memPtr);
+    else
+        return (size_t)MEM_readBE64(memPtr);
+}
+
+MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
+{
+    if (MEM_32bits())
+        MEM_writeBE32(memPtr, (U32)val);
+    else
+        MEM_writeBE64(memPtr, (U64)val);
+}
+
+
+/* function safe only for comparisons */
+MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length)
+{
+    switch (length)
+    {
+    default :
+    case 4 : return MEM_read32(memPtr);
+    case 3 : if (MEM_isLittleEndian())
+                return MEM_read32(memPtr)<<8;
+             else
+                return MEM_read32(memPtr)>>8;
+    }
+}
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif /* MEM_H_MODULE */

C/zstdmt/threading.c

@@ -0,0 +1,71 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/**
+ * This file will hold wrapper for systems, which do not support Pthreads
+ */
+
+#ifdef _WIN32
+
+/**
+ * Windows Pthread Wrapper, based on this site:
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+
+#include "threading.h"
+
+#include <process.h>
+#include <errno.h>
+
+static unsigned __stdcall worker(void *arg)
+{
+	pthread_t *thread = (pthread_t *) arg;
+	thread->arg = thread->start_routine(thread->arg);
+	return 0;
+}
+
+int
+pthread_create(pthread_t * thread, const void *unused,
+	       void *(*start_routine) (void *), void *arg)
+{
+	(void)unused;
+	thread->arg = arg;
+	thread->start_routine = start_routine;
+	thread->handle =
+	    (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
+
+	if (!thread->handle)
+		return errno;
+	else
+		return 0;
+}
+
+int _pthread_join(pthread_t * thread, void **value_ptr)
+{
+	if (!thread->handle)
+		return 0;
+
+	DWORD result = WaitForSingleObject(thread->handle, INFINITE);
+	switch (result) {
+	case WAIT_OBJECT_0:
+		if (value_ptr)
+			*value_ptr = thread->arg;
+		return 0;
+	case WAIT_ABANDONED:
+		return EINVAL;
+	default:
+		return GetLastError();
+	}
+}
+
+#endif

C/zstdmt/threading.h

@@ -0,0 +1,68 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#ifndef THREADING_H
+#define THREADING_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#ifdef _WIN32
+
+/**
+ * Windows Pthread Wrapper, based on this site:
+ * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
+ */
+
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#include <windows.h>
+
+/* mutex */
+#define pthread_mutex_t CRITICAL_SECTION
+#define pthread_mutex_init(a,b)   InitializeCriticalSection((a))
+#define pthread_mutex_destroy(a)  DeleteCriticalSection((a))
+#define pthread_mutex_lock        EnterCriticalSection
+#define pthread_mutex_unlock      LeaveCriticalSection
+
+/* pthread_create() and pthread_join() */
+typedef struct {
+	HANDLE handle;
+	void *(*start_routine) (void *);
+	void *arg;
+} pthread_t;
+
+extern int pthread_create(pthread_t * thread, const void *unused,
+			  void *(*start_routine) (void *), void *arg);
+
+#define pthread_join(a, b) _pthread_join(&(a), (b))
+extern int _pthread_join(pthread_t * thread, void **value_ptr);
+
+/**
+ * add here more systems as required
+ */
+
+#else
+
+/* POSIX Systems */
+#include <pthread.h>
+
+#endif /* POSIX Systems */
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif				/* PTHREAD_H */

C/zstdmt/zstdmt.h

@@ -0,0 +1,202 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+/* ***************************************
+ * Defines
+ ****************************************/
+
+#ifndef ZSTDMT_H
+#define ZSTDMT_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+#include <stddef.h>   /* size_t */
+
+#define ZSTDMT_THREAD_MAX 128
+#define ZSTDMT_LEVEL_MAX   22
+
+/* zstd magic values */
+#define ZSTDMT_MAGICNUMBER_V01  0x1EB52FFDU
+#define ZSTDMT_MAGICNUMBER_MIN  0xFD2FB522U
+#define ZSTDMT_MAGICNUMBER_MAX  0xFD2FB528U
+#define ZSTDMT_MAGIC_SKIPPABLE  0x184D2A50U
+
+/* **************************************
+ * Error Handling
+ ****************************************/
+
+typedef enum {
+  ZSTDMT_error_no_error,
+  ZSTDMT_error_memory_allocation,
+  ZSTDMT_error_init_missing,
+  ZSTDMT_error_read_fail,
+  ZSTDMT_error_write_fail,
+  ZSTDMT_error_data_error,
+  ZSTDMT_error_frame_compress,
+  ZSTDMT_error_frame_decompress,
+  ZSTDMT_error_compressionParameter_unsupported,
+  ZSTDMT_error_compression_library,
+  ZSTDMT_error_maxCode
+} ZSTDMT_ErrorCode;
+
+extern size_t zstdmt_errcode;
+
+#define ZSTDMT_PREFIX(name) ZSTDMT_error_##name
+#define ZSTDMT_ERROR(name) ((size_t)-ZSTDMT_PREFIX(name))
+extern unsigned ZSTDMT_isError(size_t code);
+extern const char* ZSTDMT_getErrorString(size_t code);
+
+/* **************************************
+ * Structures
+ ****************************************/
+
+typedef struct {
+	void *buf;		/* ptr to data */
+	size_t size;		/* current filled in buf */
+	size_t allocated;	/* length of buf */
+} ZSTDMT_Buffer;
+
+/**
+ * reading and writing functions
+ * - you can use stdio functions or plain read/write
+ * - just write some wrapper on your own
+ * - a sample is given in 7-Zip ZS
+ */
+typedef int (fn_read) (void *args, ZSTDMT_Buffer * in);
+typedef int (fn_write) (void *args, ZSTDMT_Buffer * out);
+
+typedef struct {
+	fn_read *fn_read;
+	void *arg_read;
+	fn_write *fn_write;
+	void *arg_write;
+} ZSTDMT_RdWr_t;
+
+/* **************************************
+ * Compression
+ ****************************************/
+
+typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
+
+/**
+ * ZSTDMT_createCCtx() - allocate new compression context
+ *
+ * This function allocates and initializes an zstd commpression context.
+ * The context can be used multiple times without the need for resetting
+ * or re-initializing.
+ *
+ * @level: compression level, which should be used (1..22)
+ * @threads: number of threads, which should be used (1..ZSTDMT_THREAD_MAX)
+ * @inputsize: - if zero, becomes some optimal value for the level
+ *             - if nonzero, the given value is taken
+ * @zstdmt_errcode: space for storing zstd errors (needed for thread safety)
+ * @return: the context on success, zero on error
+ */
+ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize);
+
+/**
+ * ZSTDMT_compressDCtx() - threaded compression for zstd
+ *
+ * This function will create valid zstd streams. The number of threads,
+ * the input chunksize and the compression level are ....
+ *
+ * @ctx: context, which needs to be created with ZSTDMT_createDCtx()
+ * @rdwr: callback structure, which defines reding/writing functions
+ * @return: zero on success, or error code
+ */
+size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx * ctx, ZSTDMT_RdWr_t * rdwr);
+
+/**
+ * ZSTDMT_GetFramesCCtx() - number of written frames
+ * ZSTDMT_GetInsizeCCtx() - read bytes of input
+ * ZSTDMT_GetOutsizeCCtx() - written bytes of output
+ *
+ * These three functions will return some statistical data of the
+ * compression context ctx.
+ *
+ * @ctx: context, which should be examined
+ * @return: the request value, or zero on error
+ */
+size_t ZSTDMT_GetFramesCCtx(ZSTDMT_CCtx * ctx);
+size_t ZSTDMT_GetInsizeCCtx(ZSTDMT_CCtx * ctx);
+size_t ZSTDMT_GetOutsizeCCtx(ZSTDMT_CCtx * ctx);
+
+/**
+ * ZSTDMT_freeCCtx() - free compression context
+ *
+ * This function will free all allocated resources, which were allocated
+ * by ZSTDMT_createCCtx(). This function can not fail.
+ *
+ * @ctx: context, which should be freed
+ */
+void ZSTDMT_freeCCtx(ZSTDMT_CCtx * ctx);
+
+/* **************************************
+ * Decompression
+ ****************************************/
+
+typedef struct ZSTDMT_DCtx_s ZSTDMT_DCtx;
+
+/**
+ * 1) allocate new cctx
+ * - return cctx or zero on error
+ *
+ * @level   - 1 .. 22
+ * @threads - 1 .. ZSTDMT_THREAD_MAX
+ * @srclen  - the max size of src for ZSTDMT_compressCCtx()
+ * @dstlen  - the min size of dst
+ */
+ZSTDMT_DCtx *ZSTDMT_createDCtx(int threads, int inputsize);
+
+/**
+ * ZSTDMT_decompressDCtx() - threaded decompression for zstd
+ *
+ * This function will decompress valid zstd streams.
+ *
+ * @ctx: context, which needs to be created with ZSTDMT_createDCtx()
+ * @rdwr: callback structure, which defines reding/writing functions
+ * @return: zero on success, or error code
+ */
+size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr);
+
+/**
+ * ZSTDMT_GetFramesDCtx() - number of read frames
+ * ZSTDMT_GetInsizeDCtx() - read bytes of input
+ * ZSTDMT_GetOutsizeDCtx() - written bytes of output
+ *
+ * These three functions will return some statistical data of the
+ * decompression context ctx.
+ *
+ * @ctx: context, which should be examined
+ * @return: the request value, or zero on error
+ */
+size_t ZSTDMT_GetFramesDCtx(ZSTDMT_DCtx * ctx);
+size_t ZSTDMT_GetInsizeDCtx(ZSTDMT_DCtx * ctx);
+size_t ZSTDMT_GetOutsizeDCtx(ZSTDMT_DCtx * ctx);
+
+/**
+ * ZSTDMT_freeDCtx() - free decompression context
+ *
+ * This function will free all allocated resources, which were allocated
+ * by ZSTDMT_createDCtx(). This function can not fail.
+ *
+ * @ctx: context, which should be freed
+ */
+void ZSTDMT_freeDCtx(ZSTDMT_DCtx * ctx);
+
+#if defined (__cplusplus)
+}
+#endif
+#endif				/* ZSTDMT_H */

C/zstdmt/zstdmt_common.c

@@ -0,0 +1,61 @@
+
+/**
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include "zstd.h"
+#include "zstdmt.h"
+
+/* ****************************************
+ * ZSTDMT Error Management
+ ******************************************/
+
+size_t zstdmt_errcode;
+
+/**
+ * ZSTDMT_isError() - tells if a return value is an error code
+ */
+unsigned ZSTDMT_isError(size_t code)
+{
+	return (code > ZSTDMT_ERROR(maxCode));
+}
+
+/**
+ * LZ4MT_getErrorString() - give error code string from function result
+ */
+const char *ZSTDMT_getErrorString(size_t code)
+{
+	if (ZSTD_isError(zstdmt_errcode))
+		return ZSTD_getErrorName(zstdmt_errcode);
+
+	static const char *notErrorCode = "Unspecified error zstmt code";
+	switch ((ZSTDMT_ErrorCode) (0 - code)) {
+	case ZSTDMT_PREFIX(no_error):
+		return "No error detected";
+	case ZSTDMT_PREFIX(memory_allocation):
+		return "Allocation error : not enough memory";
+	case ZSTDMT_PREFIX(read_fail):
+		return "Read failure";
+	case ZSTDMT_PREFIX(write_fail):
+		return "Write failure";
+	case ZSTDMT_PREFIX(data_error):
+		return "Malformed input";
+	case ZSTDMT_PREFIX(frame_compress):
+		return "Could not compress frame at once";
+	case ZSTDMT_PREFIX(frame_decompress):
+		return "Could not decompress frame at once";
+	case ZSTDMT_PREFIX(compressionParameter_unsupported):
+		return "Compression parameter is out of bound";
+	case ZSTDMT_PREFIX(compression_library):
+		return "Compression library reports failure";
+	case ZSTDMT_PREFIX(maxCode):
+	default:
+		return notErrorCode;
+	}
+}

C/zstdmt/zstdmt_compress.c

@@ -0,0 +1,427 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+#include "mem.h"
+#include "threading.h"
+#include "list.h"
+#include "zstdmt.h"
+
+/**
+ * multi threaded zstd compression
+ *
+ * - each thread works on his own
+ * - needs a callback for reading / writing
+ * - each worker does this:
+ *   1) get read mutex and read some input
+ *   2) release read mutex and do compression
+ *   3) get write mutex and write result
+ *   4) begin with step 1 again, until no input
+ */
+
+/* worker for compression */
+typedef struct {
+	ZSTDMT_CCtx *ctx;
+	pthread_t pthread;
+} cwork_t;
+
+struct writelist;
+struct writelist {
+	size_t frame;
+	ZSTDMT_Buffer out;
+	struct list_head node;
+};
+
+struct ZSTDMT_CCtx_s {
+
+	/* level: 1..ZSTDMT_LEVEL_MAX */
+	int level;
+
+	/* threads: 1..ZSTDMT_THREAD_MAX */
+	int threads;
+
+	/* buffersize for reading input */
+	int inputsize;
+
+	/* statistic */
+	size_t insize;
+	size_t outsize;
+	size_t curframe;
+	size_t frames;
+
+	/* threading */
+	cwork_t *cwork;
+
+	/* reading input */
+	pthread_mutex_t read_mutex;
+	fn_read *fn_read;
+	void *arg_read;
+
+	/* writing output */
+	pthread_mutex_t write_mutex;
+	fn_write *fn_write;
+	void *arg_write;
+
+	/* error handling */
+	pthread_mutex_t error_mutex;
+	size_t zstd_errcode;
+	size_t zstdmt_errcode;
+
+	/* lists for writing queue */
+	struct list_head writelist_free;
+	struct list_head writelist_busy;
+	struct list_head writelist_done;
+};
+
+/* **************************************
+ * Compression
+ ****************************************/
+
+ZSTDMT_CCtx *ZSTDMT_createCCtx(int threads, int level, int inputsize)
+{
+	ZSTDMT_CCtx *ctx;
+	int t;
+
+	/* allocate ctx */
+	ctx = (ZSTDMT_CCtx *) malloc(sizeof(ZSTDMT_CCtx));
+	if (!ctx)
+		return 0;
+
+	/* check threads value */
+	if (threads < 1 || threads > ZSTDMT_THREAD_MAX)
+		goto err_ctx;
+
+	/* check level */
+	if (level < 1 || level > ZSTDMT_LEVEL_MAX)
+		goto err_ctx;
+
+	/* calculate chunksize for one thread */
+	if (inputsize)
+		ctx->inputsize = inputsize;
+	else {
+		 /* XXX - windowlog */ const int mb[] = {
+			2, 2, 4, 4, 6, 6, 6,	/* 1 - 7 */
+			8, 8, 8, 8, 8, 8, 8,	/* 8 - 14 */
+			16, 16, 16, 16, 16, 16, 16, 16	/* 15 - 22 */
+		};
+		ctx->inputsize = 1024 * 1024 * mb[level - 1];
+	}
+
+	/* setup ctx */
+	ctx->level = level;
+	ctx->threads = threads;
+
+	pthread_mutex_init(&ctx->read_mutex, NULL);
+	pthread_mutex_init(&ctx->write_mutex, NULL);
+	pthread_mutex_init(&ctx->error_mutex, NULL);
+
+	INIT_LIST_HEAD(&ctx->writelist_free);
+	INIT_LIST_HEAD(&ctx->writelist_busy);
+	INIT_LIST_HEAD(&ctx->writelist_done);
+
+	ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * threads);
+	if (!ctx->cwork)
+		goto err_ctx;
+
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		w->ctx = ctx;
+	}
+
+	return ctx;
+
+err_ctx:
+	free(ctx);
+	return 0;
+}
+
+/**
+ * pt_write - queue for compressed output
+ */
+static size_t pt_write(ZSTDMT_CCtx * ctx, struct writelist *wl)
+{
+	struct list_head *entry;
+	int rv;
+
+	/* move the entry to the done list */
+	list_move(&wl->node, &ctx->writelist_done);
+
+	/* write zero byte frame (9 bytes) for type identification */
+	if (unlikely(wl->frame == 0)) {
+		unsigned char frame0[] =
+		    { 0x28, 0xB5, 0x2F, 0xFD, 0x00, 0x48, 0x01, 0x00, 0x00 };
+		ZSTDMT_Buffer b;
+		b.buf = frame0;
+		b.size = 9;
+		rv = ctx->fn_write(ctx->arg_write, &b);
+		if (rv == -1)
+			return ZSTDMT_ERROR(write_fail);
+		if (b.size != 9)
+			return ZSTDMT_ERROR(write_fail);
+		ctx->outsize += 9;
+	}
+
+	/* the entry isn't the currently needed, return...  */
+	if (wl->frame != ctx->curframe)
+		return 0;
+
+ again:
+	/* check, what can be written ... */
+	list_for_each(entry, &ctx->writelist_done) {
+		wl = list_entry(entry, struct writelist, node);
+		if (wl->frame == ctx->curframe) {
+			rv = ctx->fn_write(ctx->arg_write, &wl->out);
+			if (rv == -1)
+				return ZSTDMT_ERROR(write_fail);
+			ctx->outsize += wl->out.size;
+			ctx->curframe++;
+			list_move(entry, &ctx->writelist_free);
+			goto again;
+		}
+	}
+
+	return 0;
+}
+
+/* parallel compression worker */
+static void *pt_compress(void *arg)
+{
+	cwork_t *w = (cwork_t *) arg;
+	ZSTDMT_CCtx *ctx = w->ctx;
+	struct writelist *wl;
+	size_t result;
+	ZSTDMT_Buffer in;
+
+	/* inbuf is constant */
+	in.size = ctx->inputsize;
+	in.buf = malloc(in.size);
+	if (!in.buf)
+		return (void *)ZSTDMT_ERROR(memory_allocation);
+
+	for (;;) {
+		struct list_head *entry;
+		ZSTDMT_Buffer *out;
+		int rv;
+
+		/* allocate space for new output */
+		pthread_mutex_lock(&ctx->write_mutex);
+		if (!list_empty(&ctx->writelist_free)) {
+			/* take unused entry */
+			entry = list_first(&ctx->writelist_free);
+			wl = list_entry(entry, struct writelist, node);
+			wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;
+			list_move(entry, &ctx->writelist_busy);
+		} else {
+			/* allocate new one */
+			wl = (struct writelist *)
+			    malloc(sizeof(struct writelist));
+			if (!wl) {
+				pthread_mutex_unlock(&ctx->write_mutex);
+				free(in.buf);
+				return (void *)ZSTDMT_ERROR(memory_allocation);
+			}
+			wl->out.size = ZSTD_compressBound(ctx->inputsize) + 12;;
+			wl->out.buf = malloc(wl->out.size);
+			if (!wl->out.buf) {
+				pthread_mutex_unlock(&ctx->write_mutex);
+				free(in.buf);
+				return (void *)ZSTDMT_ERROR(memory_allocation);
+			}
+			list_add(&wl->node, &ctx->writelist_busy);
+		}
+		pthread_mutex_unlock(&ctx->write_mutex);
+		out = &wl->out;
+
+		/* read new input */
+		pthread_mutex_lock(&ctx->read_mutex);
+		in.size = ctx->inputsize;
+		rv = ctx->fn_read(ctx->arg_read, &in);
+		if (rv == -1) {
+			pthread_mutex_unlock(&ctx->read_mutex);
+			result = ZSTDMT_ERROR(read_fail);
+			goto error;
+		}
+
+		/* eof */
+		if (in.size == 0) {
+			free(in.buf);
+			pthread_mutex_unlock(&ctx->read_mutex);
+
+			pthread_mutex_lock(&ctx->write_mutex);
+			list_move(&wl->node, &ctx->writelist_free);
+			pthread_mutex_unlock(&ctx->write_mutex);
+
+			goto okay;
+		}
+		ctx->insize += in.size;
+		wl->frame = ctx->frames++;
+		pthread_mutex_unlock(&ctx->read_mutex);
+
+		/* compress whole frame */
+		{
+		unsigned char *outbuf = out->buf;
+		result = ZSTD_compress(outbuf + 12, out->size - 12, in.buf, in.size, ctx->level);
+		if (ZSTD_isError(result)) {
+			zstdmt_errcode = result;
+			result = ZSTDMT_ERROR(compression_library);
+			goto error;
+		}
+		}
+
+		/* write skippable frame */
+		{
+			unsigned char *outbuf = out->buf;
+
+			MEM_writeLE32(outbuf + 0, ZSTDMT_MAGIC_SKIPPABLE);
+			MEM_writeLE32(outbuf + 4, 4);
+			MEM_writeLE32(outbuf + 8, (U32) result);
+			out->size = result + 12;
+		}
+
+		/* write result */
+		pthread_mutex_lock(&ctx->write_mutex);
+		result = pt_write(ctx, wl);
+		pthread_mutex_unlock(&ctx->write_mutex);
+		if (ZSTDMT_isError(result))
+			goto error;
+	}
+
+ okay:
+	return 0;
+ error:
+	pthread_mutex_lock(&ctx->write_mutex);
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	return (void *)result;
+}
+
+/* compress data, until input ends */
+size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx * ctx, ZSTDMT_RdWr_t * rdwr)
+{
+	int t;
+	void *rv = 0;
+
+	if (!ctx)
+		return ZSTDMT_ERROR(init_missing);
+
+	/* setup reading and writing functions */
+	ctx->fn_read = rdwr->fn_read;
+	ctx->fn_write = rdwr->fn_write;
+	ctx->arg_read = rdwr->arg_read;
+	ctx->arg_write = rdwr->arg_write;
+
+	/* init counter and error codes */
+	ctx->insize = 0;
+	ctx->outsize = 0;
+	ctx->frames = 0;
+	ctx->curframe = 0;
+	ctx->zstdmt_errcode = 0;
+
+	/* start all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		pthread_create(&w->pthread, NULL, pt_compress, w);
+	}
+
+	/* wait for all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		void *p;
+		pthread_join(w->pthread, &p);
+		if (p)
+			rv = p;
+	}
+
+	/* clean up the free list */
+	while (!list_empty(&ctx->writelist_free)) {
+		struct writelist *wl;
+		struct list_head *entry;
+		entry = list_first(&ctx->writelist_free);
+		wl = list_entry(entry, struct writelist, node);
+		free(wl->out.buf);
+		list_del(&wl->node);
+		free(wl);
+	}
+
+	/* on error, these two lists may have some entries */
+	if (rv) {
+		struct writelist *wl;
+		struct list_head *entry;
+
+		while (!list_empty(&ctx->writelist_busy)) {
+			entry = list_first(&ctx->writelist_busy);
+			wl = list_entry(entry, struct writelist, node);
+			free(wl->out.buf);
+			list_del(&wl->node);
+			free(wl);
+		}
+
+		while (!list_empty(&ctx->writelist_done)) {
+			entry = list_first(&ctx->writelist_done);
+			wl = list_entry(entry, struct writelist, node);
+			free(wl->out.buf);
+			list_del(&wl->node);
+			free(wl);
+		}
+	}
+
+	return (size_t) rv;
+}
+
+/* returns current uncompressed data size */
+size_t ZSTDMT_GetInsizeCCtx(ZSTDMT_CCtx * ctx)
+{
+	if (!ctx)
+		return ZSTDMT_ERROR(init_missing);
+
+	/* no mutex needed here */
+	return ctx->insize;
+}
+
+/* returns the current compressed data size */
+size_t ZSTDMT_GetOutsizeCCtx(ZSTDMT_CCtx * ctx)
+{
+	if (!ctx)
+		return ZSTDMT_ERROR(init_missing);
+
+	/* no mutex needed here */
+	return ctx->outsize;
+}
+
+/* returns the current compressed data frame count */
+size_t ZSTDMT_GetFramesCCtx(ZSTDMT_CCtx * ctx)
+{
+	if (!ctx)
+		return ZSTDMT_ERROR(init_missing);
+
+	/* no mutex needed here */
+	return ctx->curframe;
+}
+
+/* free all allocated buffers and structures */
+void ZSTDMT_freeCCtx(ZSTDMT_CCtx * ctx)
+{
+	if (!ctx)
+		return;
+
+	free(ctx->cwork);
+	free(ctx);
+	ctx = 0;
+
+	return;
+}

C/zstdmt/zstdmt_decompress.c

@@ -0,0 +1,857 @@
+
+/**
+ * Copyright (c) 2016 Tino Reichardt
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ *
+ * You can contact the author at:
+ * - zstdmt source repository: https://github.com/mcmilk/zstdmt
+ */
+
+#include <stdlib.h>
+#include <string.h>
+
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+#include "mem.h"
+#include "threading.h"
+#include "list.h"
+#include "zstdmt.h"
+
+/**
+ * multi threaded zstd decompression
+ *
+ * - each thread works on his own
+ * - needs a callback for reading / writing
+ * - each worker does this:
+ *   1) get read mutex and read some input
+ *   2) release read mutex and do decompression
+ *   3) get write mutex and write result
+ *   4) begin with step 1 again, until no input
+ */
+
+#if 0
+#include <stdio.h>
+#define dprintf(fmt, arg...) do { printf(fmt, ## arg); } while (0)
+#else
+#define dprintf(fmt, ...)
+#endif				/* DEBUG */
+
+extern size_t zstdmt_errcode;
+
+/* worker for compression */
+typedef struct {
+	ZSTDMT_DCtx *ctx;
+	pthread_t pthread;
+	ZSTDMT_Buffer in;
+	ZSTD_DStream *dctx;
+} cwork_t;
+
+struct writelist;
+struct writelist {
+	size_t frame;
+	ZSTDMT_Buffer out;
+	struct list_head node;
+};
+
+struct ZSTDMT_DCtx_s {
+
+	/* threads: 1..ZSTDMT_THREAD_MAX */
+	int threads;
+	int threadswanted;
+
+	/* input buffer, used at single threading */
+	size_t inputsize;
+
+	/* buffersize used for output */
+	size_t outputsize;
+
+	/* statistic */
+	size_t insize;
+	size_t outsize;
+	size_t curframe;
+	size_t frames;
+
+	/* threading */
+	cwork_t *cwork;
+
+	/* reading input */
+	pthread_mutex_t read_mutex;
+	fn_read *fn_read;
+	void *arg_read;
+
+	/* writing output */
+	pthread_mutex_t write_mutex;
+	fn_write *fn_write;
+	void *arg_write;
+
+	/* error handling */
+	pthread_mutex_t error_mutex;
+
+	/* lists for writing queue */
+	struct list_head writelist_free;
+	struct list_head writelist_busy;
+	struct list_head writelist_done;
+};
+
+/* **************************************
+ * Decompression
+ ****************************************/
+
+ZSTDMT_DCtx *ZSTDMT_createDCtx(int threads, int inputsize)
+{
+	ZSTDMT_DCtx *ctx;
+
+	/* allocate ctx */
+	ctx = (ZSTDMT_DCtx *) malloc(sizeof(ZSTDMT_DCtx));
+	if (!ctx)
+		return 0;
+
+	/* check threads value */
+	if (threads < 1 || threads > ZSTDMT_THREAD_MAX)
+		return 0;
+
+	/* setup ctx */
+	ctx->threadswanted = threads;
+	ctx->threads = 0;
+	ctx->insize = 0;
+	ctx->outsize = 0;
+	ctx->frames = 0;
+	ctx->curframe = 0;
+
+	/* will be used for single stream only */
+	if (inputsize)
+		ctx->inputsize = inputsize;
+	else
+		ctx->inputsize = 1024 * 512;
+
+	/* frame size (will get higher, when needed) */
+	ctx->outputsize = 1024 * 512;
+
+	/* later */
+	ctx->cwork = 0;
+
+	return ctx;
+}
+
+/**
+ * IsZstd_Magic - check, if 4 bytes are valid ZSTD MAGIC
+ */
+static int IsZstd_Magic(unsigned char *buf)
+{
+	U32 magic = MEM_readLE32(buf);
+	if (magic == ZSTDMT_MAGICNUMBER_V01)
+		return 1;
+	return (magic >= ZSTDMT_MAGICNUMBER_MIN
+		&& magic <= ZSTDMT_MAGICNUMBER_MAX);
+}
+
+/**
+ * IsZstd_Skippable - check, if 4 bytes are MAGIC_SKIPPABLE
+ */
+static int IsZstd_Skippable(unsigned char *buf)
+{
+	return (MEM_readLE32(buf) == ZSTDMT_MAGIC_SKIPPABLE);
+}
+
+/**
+ * pt_write - queue for decompressed output
+ */
+static size_t pt_write(ZSTDMT_DCtx * ctx, struct writelist *wl)
+{
+	struct list_head *entry;
+
+	/* move the entry to the done list */
+	list_move(&wl->node, &ctx->writelist_done);
+ again:
+	/* check, what can be written ... */
+	list_for_each(entry, &ctx->writelist_done) {
+		wl = list_entry(entry, struct writelist, node);
+		if (wl->frame == ctx->curframe) {
+			int rv = ctx->fn_write(ctx->arg_write, &wl->out);
+			if (rv == -1)
+				return ZSTDMT_ERROR(write_fail);
+			ctx->outsize += wl->out.size;
+			ctx->curframe++;
+			list_move(entry, &ctx->writelist_free);
+			goto again;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * pt_read - read compressed input
+ */
+static size_t pt_read(ZSTDMT_DCtx * ctx, ZSTDMT_Buffer * in, size_t * frame)
+{
+	unsigned char hdrbuf[12];
+	ZSTDMT_Buffer hdr;
+	size_t toRead;
+	int rv;
+
+	pthread_mutex_lock(&ctx->read_mutex);
+
+	/* special case, some bytes were read by magic check */
+	if (unlikely(ctx->frames == 0)) {
+		/* the magic check reads exactly 16 bytes! */
+		if (unlikely(in->size != 16))
+			goto error_data;
+		ctx->insize += 16;
+
+		/**
+		 * zstdmt mode, with zstd magic prefix
+		 * 9 bytes zero byte frame + 12 byte skippable
+		 * - 21 bytes to read, 16 bytes done
+		 * - read 5 bytes, put them together (12 byte hdr)
+		 */
+		if (!IsZstd_Skippable(in->buf)) {
+			memcpy(hdrbuf, in->buf, 7);
+			hdr.buf = hdrbuf + 7;
+			hdr.size = 5;
+			rv = ctx->fn_read(ctx->arg_read, &hdr);
+			if (rv == -1)
+				goto error_read;
+			if (hdr.size != 5)
+				goto error_data;
+			hdr.buf = hdrbuf;
+			ctx->insize += 16 + 5;
+
+			/* read data */
+			toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
+			in->size = toRead;
+			in->buf = malloc(in->size);
+			if (!in->buf)
+				goto error_nomem;
+			in->allocated = in->size;
+			rv = ctx->fn_read(ctx->arg_read, in);
+			if (rv == -1)
+				goto error_read;
+			if (in->size != toRead)
+				goto error_data;
+			ctx->insize += in->size;
+			*frame = ctx->frames++;
+			pthread_mutex_unlock(&ctx->read_mutex);
+			return 0;	/* done! */
+		}
+
+		/**
+		 * pzstd mode, no prefix
+		 * - start directly with 12 byte skippable frame
+		 */
+		if (IsZstd_Skippable(in->buf)) {
+			unsigned char *start = in->buf;	/* 16 bytes data */
+			toRead = MEM_readLE32((unsigned char *)start + 8);
+			in->size = toRead;
+			in->buf = malloc(in->size);
+			if (!in->buf)
+				goto error_nomem;
+			in->allocated = in->size;
+			/* copy 4 bytes user data to new buf */
+			memcpy(in->buf, start + 12, 4);
+			start = in->buf;	/* point to in->buf now */
+
+			/* 12 byte skippable, so 4 bytes data done */
+			in->buf = start + 4;
+			in->size = toRead - 4;
+			rv = ctx->fn_read(ctx->arg_read, in);
+			if (rv == -1)
+				goto error_read;
+			if (in->size != toRead - 4)
+				goto error_data;
+			ctx->insize += in->size;
+			in->buf = start;	/* restore inbuf */
+			in->size += 4;
+			*frame = ctx->frames++;
+			pthread_mutex_unlock(&ctx->read_mutex);
+			return 0;	/* done! */
+		}
+	}
+
+	/**
+	 * read next skippable frame (12 bytes)
+	 * 4 bytes skippable magic
+	 * 4 bytes little endian, must be: 4 (user data size)
+	 * 4 bytes little endian, size to read (user data)
+	 */
+	hdr.buf = hdrbuf;
+	hdr.size = 12;
+	rv = ctx->fn_read(ctx->arg_read, &hdr);
+	if (rv == -1)
+		goto error_read;
+
+	/* eof reached ? */
+	if (unlikely(hdr.size == 0)) {
+		pthread_mutex_unlock(&ctx->read_mutex);
+		in->size = 0;
+		return 0;
+	}
+
+	/* check header data */
+	if (unlikely(hdr.size != 12))
+		goto error_read;
+	if (unlikely(!IsZstd_Skippable(hdr.buf)))
+		goto error_data;
+	ctx->insize += 12;
+
+	/* read new input (size should be _toRead_ bytes */
+	toRead = MEM_readLE32((unsigned char *)hdr.buf + 8);
+	{
+		if (in->allocated < toRead) {
+			/* need bigger input buffer */
+			if (in->allocated)
+				in->buf = realloc(in->buf, toRead);
+			else
+				in->buf = malloc(toRead);
+			if (!in->buf)
+				goto error_nomem;
+			in->allocated = toRead;
+		}
+
+		in->size = toRead;
+		rv = ctx->fn_read(ctx->arg_read, in);
+		/* generic read failure! */
+		if (rv == -1)
+			goto error_read;
+		/* needed more bytes! */
+		if (in->size != toRead)
+			goto error_data;
+
+		ctx->insize += in->size;
+	}
+	*frame = ctx->frames++;
+	pthread_mutex_unlock(&ctx->read_mutex);
+
+	/* done, no error */
+	return 0;
+
+ error_data:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ZSTDMT_ERROR(data_error);
+ error_read:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ZSTDMT_ERROR(read_fail);
+ error_nomem:
+	pthread_mutex_unlock(&ctx->read_mutex);
+	return ZSTDMT_ERROR(memory_allocation);
+}
+
+static void *pt_decompress(void *arg)
+{
+	cwork_t *w = (cwork_t *) arg;
+	ZSTDMT_Buffer *in = &w->in;
+	ZSTDMT_DCtx *ctx = w->ctx;
+	struct writelist *wl;
+	size_t result = 0;
+	ZSTDMT_Buffer collect;
+
+		/* init dstream stream */
+		result = ZSTD_initDStream(w->dctx);
+		if (ZSTD_isError(result)) {
+			zstdmt_errcode = result;
+			return (void *)ZSTDMT_ERROR(compression_library);
+		}
+
+	collect.buf = 0;
+	collect.size = 0;
+	collect.allocated = 0;
+	for (;;) {
+		ZSTDMT_Buffer *out;
+		ZSTD_inBuffer zIn;
+		ZSTD_outBuffer zOut;
+
+		/* select or allocate space for new output */
+		pthread_mutex_lock(&ctx->write_mutex);
+		if (!list_empty(&ctx->writelist_free)) {
+			/* take unused entry */
+			struct list_head *entry;
+			entry = list_first(&ctx->writelist_free);
+			wl = list_entry(entry, struct writelist, node);
+			list_move(entry, &ctx->writelist_busy);
+		} else {
+			/* allocate new one */
+			wl = (struct writelist *)
+			    malloc(sizeof(struct writelist));
+			if (!wl) {
+				result = ZSTDMT_ERROR(memory_allocation);
+				goto error_unlock;
+			}
+			out = &wl->out;
+			out->size = ctx->outputsize;
+			out->buf = malloc(out->size);
+			if (!out->buf) {
+				result = ZSTDMT_ERROR(memory_allocation);
+				goto error_unlock;
+			}
+			out->allocated = out->size;
+			list_add(&wl->node, &ctx->writelist_busy);
+		}
+
+		/* start with 512KB */
+		/* XXX, add framesize detection... */
+		out = &wl->out;
+		pthread_mutex_unlock(&ctx->write_mutex);
+
+		/* init dstream stream */
+		result = ZSTD_resetDStream(w->dctx);
+		if (ZSTD_isError(result)) {
+			zstdmt_errcode = result;
+			return (void *)ZSTDMT_ERROR(compression_library);
+		}
+
+		/* zero should not happen here! */
+		result = pt_read(ctx, in, &wl->frame);
+		if (in->size == 0)
+			break;
+		if (ZSTDMT_isError(result)) {
+			goto error_lock;
+		}
+
+		zIn.size = in->allocated;
+		zIn.src = in->buf;
+		zIn.pos = 0;
+
+		for (;;) {
+ again:
+			/* decompress loop */
+			zOut.size = out->allocated;
+			zOut.dst = out->buf;
+			zOut.pos = 0;
+
+			dprintf
+			    ("ZSTD_decompressStream() zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n",
+			     zIn.size, zIn.pos, zOut.size, zOut.pos);
+			result = ZSTD_decompressStream(w->dctx, &zOut, &zIn);
+			dprintf
+			    ("ZSTD_decompressStream(), ret=%zu zIn.size=%zu zIn.pos=%zu zOut.size=%zu zOut.pos=%zu\n",
+			     result, zIn.size, zIn.pos, zOut.size, zOut.pos);
+			if (ZSTD_isError(result))
+				goto error_clib;
+
+			/* end of frame */
+			if (result == 0) {
+				/* put collected stuff together */
+				if (collect.size) {
+					void *bnew;
+					bnew = malloc(collect.size + zOut.pos);
+					if (!bnew) {
+						result =
+						    ZSTDMT_ERROR(memory_allocation);
+						goto error_lock;
+					}
+					memcpy((char *)bnew, collect.buf,
+					       collect.size);
+					memcpy((char *)bnew + collect.size,
+					       out->buf, zOut.pos);
+					free(collect.buf);
+					free(out->buf);
+					out->buf = bnew;
+					out->size = collect.size + zOut.pos;
+					out->allocated = out->size;
+					collect.buf = 0;
+					collect.size = 0;
+				} else {
+					out->size = zOut.pos;
+				}
+				/* write result */
+				pthread_mutex_lock(&ctx->write_mutex);
+				result = pt_write(ctx, wl);
+				if (ZSTDMT_isError(result))
+					goto error_unlock;
+				pthread_mutex_unlock(&ctx->write_mutex);
+				/* will read next input */
+				break;
+			}
+
+			/* out buffer to small for full frame */
+			if (result != 0) {
+				/* collect old content from out */
+				collect.buf =
+				    realloc(collect.buf,
+					    collect.size + out->size);
+				memcpy((char *)collect.buf + collect.size,
+				       out->buf, out->size);
+				collect.size = collect.size + out->size;
+
+				/* double the buffer, until it fits */
+				pthread_mutex_lock(&ctx->write_mutex);
+				out->size *= 2;
+				ctx->outputsize = out->size;
+				pthread_mutex_unlock(&ctx->write_mutex);
+				out->buf = realloc(out->buf, out->size);
+				if (!out->buf) {
+					result = ZSTDMT_ERROR(memory_allocation);
+					goto error_lock;
+				}
+				out->allocated = out->size;
+				goto again;
+			}
+
+			if (zIn.pos == zIn.size)
+				break;	/* should fail... */
+		}		/* decompress loop */
+	}			/* read input loop */
+
+	/* everything is okay */
+	pthread_mutex_lock(&ctx->write_mutex);
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	if (in->allocated)
+		free(in->buf);
+	return 0;
+
+ error_clib:
+	zstdmt_errcode = result;
+	result = ZSTDMT_ERROR(compression_library);
+	/* fall through */
+ error_lock:
+	pthread_mutex_lock(&ctx->write_mutex);
+ error_unlock:
+	list_move(&wl->node, &ctx->writelist_free);
+	pthread_mutex_unlock(&ctx->write_mutex);
+	if (in->allocated)
+		free(in->buf);
+	return (void *)result;
+}
+
+/* single threaded */
+static size_t st_decompress(void *arg)
+{
+	ZSTDMT_DCtx *ctx = (ZSTDMT_DCtx *) arg;
+	cwork_t *w = &ctx->cwork[0];
+	ZSTDMT_Buffer In, Out;
+	ZSTDMT_Buffer *in = &In;
+	ZSTDMT_Buffer *out = &Out;
+	ZSTDMT_Buffer *magic = &w->in;
+	size_t result;
+	int rv;
+
+	ZSTD_inBuffer zIn;
+	ZSTD_outBuffer zOut;
+
+	/* init dstream stream */
+	result = ZSTD_initDStream(w->dctx);
+	if (ZSTD_isError(result)) {
+		zstdmt_errcode = result;
+		return ZSTDMT_ERROR(compression_library);
+	}
+
+	/* allocate space for input buffer */
+	in->size = ZSTD_DStreamInSize();
+	in->buf = malloc(in->size);
+	if (!in->buf)
+		return ZSTDMT_ERROR(memory_allocation);
+	in->allocated = in->size;
+
+	/* allocate space for output buffer */
+	out->size = ZSTD_DStreamOutSize();
+	out->buf = malloc(out->size);
+	if (!out->buf) {
+		free(in->buf);
+		return ZSTDMT_ERROR(memory_allocation);
+	}
+	out->allocated = out->size;
+
+	/* we read already some bytes, handle that: */
+	{
+		/* remember in->buf */
+		unsigned char *buf = in->buf;
+
+		/* fill first read bytes to buffer... */
+		memcpy(in->buf, magic->buf, magic->size);
+		magic->buf = in->buf;
+		in->buf = buf + magic->size;
+		in->size = in->allocated - magic->size;
+
+		/* read more bytes, to fill buffer */
+		rv = ctx->fn_read(ctx->arg_read, in);
+		if (rv == -1) {
+			result = ZSTDMT_ERROR(read_fail);
+			goto error;
+		}
+
+		/* ready, first buffer complete */
+		in->buf = buf;
+		in->size += magic->size;
+		ctx->insize += in->size;
+	}
+
+	zIn.src = in->buf;
+	zIn.size = in->size;
+	zIn.pos = 0;
+
+	zOut.dst = out->buf;
+
+	for (;;) {
+		for (;;) {
+			/* decompress loop */
+			zOut.size = out->allocated;
+			zOut.pos = 0;
+
+			result = ZSTD_decompressStream(w->dctx, &zOut, &zIn);
+			if (ZSTD_isError(result))
+				goto error_clib;
+
+			if (zOut.pos) {
+				ZSTDMT_Buffer w;
+				w.size = zOut.pos;
+				w.buf = zOut.dst;
+				rv = ctx->fn_write(ctx->arg_write, &w);
+				ctx->outsize += zOut.pos;
+			}
+
+			/* one more round */
+			if ((zIn.pos == zIn.size) && (result == 1) && zOut.pos)
+				continue;
+
+			/* finished */
+			if (zIn.pos == zIn.size)
+				break;
+
+			/* end of frame */
+			if (result == 0) {
+				result = ZSTD_resetDStream(w->dctx);
+				if (ZSTD_isError(result))
+					goto error_clib;
+			}
+		}		/* decompress */
+
+		/* read next input */
+		in->size = in->allocated;
+		rv = ctx->fn_read(ctx->arg_read, in);
+		if (rv == -1) {
+			result = ZSTDMT_ERROR(read_fail);
+			goto error;
+		}
+
+		if (in->size == 0)
+			goto okay;
+		ctx->insize += in->size;
+
+		zIn.size = in->size;
+		zIn.pos = 0;
+	}			/* read */
+
+ error_clib:
+	zstdmt_errcode = result;
+	result = ZSTDMT_ERROR(compression_library);
+	/* fall through */
+ error:
+	/* return with error */
+	free(out->buf);
+	free(in->buf);
+	return result;
+ okay:
+	/* no error */
+	free(out->buf);
+	free(in->buf);
+	return 0;
+}
+
+#define TYPE_UNKNOWN       0
+#define TYPE_SINGLE_THREAD 1
+#define TYPE_MULTI_THREAD  2
+
+size_t ZSTDMT_decompressDCtx(ZSTDMT_DCtx * ctx, ZSTDMT_RdWr_t * rdwr)
+{
+	unsigned char buf[16];
+	ZSTDMT_Buffer In;
+	ZSTDMT_Buffer *in = &In;
+	cwork_t *w;
+	int t, rv, type = TYPE_UNKNOWN;
+
+	if (!ctx)
+		return ZSTDMT_ERROR(compressionParameter_unsupported);
+
+	/* init reading and writing functions */
+	ctx->fn_read = rdwr->fn_read;
+	ctx->fn_write = rdwr->fn_write;
+	ctx->arg_read = rdwr->arg_read;
+	ctx->arg_write = rdwr->arg_write;
+
+	/**
+	 * possible valid magic's for us, we need 16 bytes, for checking
+	 *
+	 * 1) ZSTDMT_MAGIC @0 -> ST Stream
+	 * 2) ZSTDMT_MAGIC @0 + MAGIC_SKIPPABLE @9 -> MT Stream else ST
+	 * 3) MAGIC_SKIPPABLE @0 + ZSTDMT_MAGIC @12 -> MT Stream
+	 * 4) all other: not valid!
+	 */
+
+	/* check for ZSTDMT_MAGIC_SKIPPABLE */
+	in->buf = buf;
+	in->size = 16;
+	rv = ctx->fn_read(ctx->arg_read, in);
+	if (rv == -1)
+		return ZSTDMT_ERROR(read_fail);
+
+	/* must be single threaded standard zstd, when smaller 16 bytes */
+	if (in->size < 16) {
+		if (!IsZstd_Magic(buf))
+			return ZSTDMT_ERROR(data_error);
+		dprintf("single thread style, current pos=%zu\n", in->size);
+		type = TYPE_SINGLE_THREAD;
+		if (in->size == 9) {
+			/* create empty file */
+			ctx->threads = 0;
+			ctx->cwork = 0;
+			return 0;
+		}
+	} else {
+		if (IsZstd_Skippable(buf) && IsZstd_Magic(buf + 12)) {
+			/* pzstd */
+			dprintf("pzstd style\n");
+			type = TYPE_MULTI_THREAD;
+		} else if (IsZstd_Magic(buf) && IsZstd_Skippable(buf + 9)) {
+			/* zstdmt */
+			dprintf("zstdmt style\n");
+			type = TYPE_MULTI_THREAD;
+			/* set buffer to the */
+		} else if (IsZstd_Magic(buf)) {
+			/* some std zstd stream */
+			dprintf("single thread style, current pos=%zu\n",
+				in->size);
+			type = TYPE_SINGLE_THREAD;
+		} else {
+			/* invalid */
+			dprintf("not valid\n");
+			return ZSTDMT_ERROR(data_error);
+		}
+	}
+
+	/* use single thread extraction, when only one thread is there */
+	if (ctx->threadswanted == 1)
+		type = TYPE_SINGLE_THREAD;
+
+	/* single threaded, but with known sizes */
+	if (type == TYPE_SINGLE_THREAD) {
+		ctx->threads = 1;
+		ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t));
+		if (!ctx->cwork)
+			return ZSTDMT_ERROR(memory_allocation);
+		w = &ctx->cwork[0];
+		w->in.buf = in->buf;
+		w->in.size = in->size;
+		w->in.allocated = 0;
+		w->ctx = ctx;
+		w->dctx = ZSTD_createDStream();
+		if (!w->dctx)
+			return ZSTDMT_ERROR(memory_allocation);
+
+		/* test, if pt_decompress is better... */
+		return st_decompress(ctx);
+	}
+
+	/* setup thread work */
+	ctx->threads = ctx->threadswanted;
+	ctx->cwork = (cwork_t *) malloc(sizeof(cwork_t) * ctx->threads);
+	if (!ctx->cwork)
+		return ZSTDMT_ERROR(memory_allocation);
+
+	for (t = 0; t < ctx->threads; t++) {
+		w = &ctx->cwork[t];
+		/* one of the threads must reuse the first bytes */
+		w->in.buf = in->buf;
+		w->in.size = in->size;
+		w->in.allocated = 0;
+		w->ctx = ctx;
+		w->dctx = ZSTD_createDStream();
+		if (!w->dctx)
+			return ZSTDMT_ERROR(memory_allocation);
+	}
+
+	/* real multi threaded, init pthread's */
+	pthread_mutex_init(&ctx->read_mutex, NULL);
+	pthread_mutex_init(&ctx->write_mutex, NULL);
+	pthread_mutex_init(&ctx->error_mutex, NULL);
+
+	INIT_LIST_HEAD(&ctx->writelist_free);
+	INIT_LIST_HEAD(&ctx->writelist_busy);
+	INIT_LIST_HEAD(&ctx->writelist_done);
+
+	/* multi threaded */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		pthread_create(&w->pthread, NULL, pt_decompress, w);
+	}
+
+	/* wait for all workers */
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		void *p;
+		pthread_join(w->pthread, &p);
+		if (p)
+			return (size_t) p;
+	}
+
+	/* clean up pthread stuff */
+	pthread_mutex_destroy(&ctx->read_mutex);
+	pthread_mutex_destroy(&ctx->write_mutex);
+	pthread_mutex_destroy(&ctx->error_mutex);
+
+	/* clean up the buffers */
+	while (!list_empty(&ctx->writelist_free)) {
+		struct writelist *wl;
+		struct list_head *entry;
+		entry = list_first(&ctx->writelist_free);
+		wl = list_entry(entry, struct writelist, node);
+		free(wl->out.buf);
+		list_del(&wl->node);
+		free(wl);
+	}
+
+	return 0;
+}
+
+/* returns current uncompressed data size */
+size_t ZSTDMT_GetInsizeDCtx(ZSTDMT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->insize;
+}
+
+/* returns the current compressed data size */
+size_t ZSTDMT_GetOutsizeDCtx(ZSTDMT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->outsize;
+}
+
+/* returns the current compressed frames */
+size_t ZSTDMT_GetFramesDCtx(ZSTDMT_DCtx * ctx)
+{
+	if (!ctx)
+		return 0;
+
+	return ctx->curframe;
+}
+
+void ZSTDMT_freeDCtx(ZSTDMT_DCtx * ctx)
+{
+	int t;
+
+	if (!ctx)
+		return;
+
+	for (t = 0; t < ctx->threads; t++) {
+		cwork_t *w = &ctx->cwork[t];
+		ZSTD_freeDStream(w->dctx);
+	}
+
+	if (ctx->cwork)
+		free(ctx->cwork);
+
+	free(ctx);
+	ctx = 0;
+
+	return;
+}

CPP/7zip/7zip.mak

@@ -27,7 +27,10 @@ OBJS = \
   $(COMPRESS_OBJS) \
   $(CRYPTO_OBJS) \
   $(C_OBJS) \
+  $(LZ4_OBJS) \
+  $(LZ5_OBJS) \
   $(ZSTD_OBJS) \
+  $(ZSTDMT_OBJS) \
   $(ASM_OBJS) \
   $O\resource.res \
 
@@ -173,11 +176,26 @@ $(C_OBJS): ../../../../C/$(*B).c
 	$(COMPL_O2)
 !ENDIF
 
+!IFDEF LZ4_OBJS
+$(LZ4_OBJS): ../../../../C/lz4/$(*B).c
+	$(COMPL_O2)
+!ENDIF
+
+!IFDEF LZ5_OBJS
+$(LZ5_OBJS): ../../../../C/lz5/$(*B).c
+	$(COMPL_O2)
+!ENDIF
+
 !IFDEF ZSTD_OBJS
 $(ZSTD_OBJS): ../../../../C/zstd/$(*B).c
 	$(COMPL_O2)
 !ENDIF
 
+!IFDEF ZSTDMT_OBJS
+$(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c
+	$(COMPL_O2)
+!ENDIF
+
 
 !ELSE
 
@@ -240,8 +258,17 @@ $(ZSTD_OBJS): ../../../../C/zstd/$(*B).c
 	$(COMPLB_O2)
 {../../../../C}.c{$O}.obj::
 	$(CCOMPLB)
+{../../../../C/lz4}.c{$O}.obj::
+	$(CCOMPLB)
+{../../../../C/lz5}.c{$O}.obj::
+	$(CCOMPLB)
 {../../../../C/zstd}.c{$O}.obj::
 	$(CCOMPLB)
+{../../../../C/zstdmt}.c{$O}.obj::
+	$(CCOMPLB) \
+	-I ../../../../C/lz4 \
+	-I ../../../../C/lz5 \
+	-I ../../../../C/zstd
 
 !ENDIF
 

CPP/7zip/Archive/7z/7zUpdate.cpp

@@ -558,7 +558,7 @@ static int CompareEmptyItems(const unsigned *p1, const unsigned *p2, void *param
 }
 
 static const char *g_Exts =
-  " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lzh lzo lzx pak rar rpm sit zoo"
+  " 7z xz lzma ace arc arj bz tbz bz2 tbz2 cab deb gz tgz ha lha lz tlz lz4 tlz4 lz5 tlz5 lzh lzo lzx pak rar rpm sit zoo zst"
   " zip jar ear war msi"
   " 3gp avi mov mpeg mpg mpe wmv"
   " aac ape fla flac la mp3 m4a mp4 ofr ogg pac ra rm rka shn swa tta wv wma wav"

CPP/7zip/Archive/IArchive.h

@@ -155,7 +155,7 @@ IArchiveExtractCallback::GetStream()
 if (IProgress::SetTotal() was called)
 {
   IProgress::SetCompleted(completeValue) uses
-    packSize   - for some stream formats (xz, gz, bz2, lzma, z, ppmd).
+    packSize   - for some stream formats (xz, gz, bz2, lz, lzma, z, ppmd).
     unpackSize - for another formats.
 }
 else

CPP/7zip/Archive/Lz4Handler.cpp

@@ -0,0 +1,366 @@
+// ZstdHandler.cpp
+
+#include "StdAfx.h"
+
+#include "../../../C/CpuArch.h"
+#include "../../Common/ComTry.h"
+#include "../../Common/Defs.h"
+
+#include "../Common/ProgressUtils.h"
+#include "../Common/RegisterArc.h"
+#include "../Common/StreamUtils.h"
+
+#include "../Compress/Lz4Decoder.h"
+#include "../Compress/Lz4Encoder.h"
+#include "../Compress/CopyCoder.h"
+
+#include "Common/DummyOutStream.h"
+#include "Common/HandlerOut.h"
+
+using namespace NWindows;
+
+namespace NArchive {
+namespace NLZ4 {
+
+class CHandler:
+  public IInArchive,
+  public IArchiveOpenSeq,
+  public IOutArchive,
+  public ISetProperties,
+  public CMyUnknownImp
+{
+  CMyComPtr<IInStream> _stream;
+  CMyComPtr<ISequentialInStream> _seqStream;
+
+  bool _isArc;
+  bool _dataAfterEnd;
+  bool _needMoreInput;
+
+  bool _packSize_Defined;
+  bool _unpackSize_Defined;
+
+  UInt64 _packSize;
+  UInt64 _unpackSize;
+  UInt64 _numStreams;
+  UInt64 _numBlocks;
+
+  CSingleMethodProps _props;
+
+public:
+  MY_UNKNOWN_IMP4(
+      IInArchive,
+      IArchiveOpenSeq,
+      IOutArchive,
+      ISetProperties)
+  INTERFACE_IInArchive(;)
+  INTERFACE_IOutArchive(;)
+  STDMETHOD(OpenSeq)(ISequentialInStream *stream);
+  STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
+
+  CHandler() { }
+};
+
+static const Byte kProps[] =
+{
+  kpidSize,
+  kpidPackSize
+};
+
+static const Byte kArcProps[] =
+{
+  kpidNumStreams,
+  kpidNumBlocks
+};
+
+IMP_IInArchive_Props
+IMP_IInArchive_ArcProps
+
+STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/)
+{
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
+{
+  *numItems = 1;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
+{
+  NCOM::CPropVariant prop;
+  switch (propID)
+  {
+    case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
+    case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
+  }
+  prop.Detach(value);
+  return S_OK;
+}
+
+static const unsigned kSignatureCheckSize = 4;
+
+API_FUNC_static_IsArc IsArc_lz4(const Byte *p, size_t size)
+{
+  if (size < 4)
+    return k_IsArc_Res_NEED_MORE;
+
+  UInt32 magic = GetUi32(p);
+
+  // skippable frames
+  if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) {
+    if (size < 16)
+      return k_IsArc_Res_NEED_MORE;
+    magic = GetUi32(p+12);
+  }
+
+  // lz4 magic
+  if (magic == 0x184D2204)
+    return k_IsArc_Res_YES;
+
+  return k_IsArc_Res_NO;
+}
+}
+
+STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)
+{
+  COM_TRY_BEGIN
+  Close();
+  {
+    Byte buf[kSignatureCheckSize];
+    RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize));
+    if (IsArc_lz4(buf, kSignatureCheckSize) == k_IsArc_Res_NO)
+      return S_FALSE;
+    _isArc = true;
+    _stream = stream;
+    _seqStream = stream;
+    RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
+  }
+  return S_OK;
+  COM_TRY_END
+}
+
+STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
+{
+  Close();
+  _isArc = true;
+  _seqStream = stream;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::Close()
+{
+  _isArc = false;
+  _dataAfterEnd = false;
+  _needMoreInput = false;
+
+  _packSize_Defined = false;
+  _unpackSize_Defined = false;
+
+  _packSize = 0;
+
+  _seqStream.Release();
+  _stream.Release();
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
+    Int32 testMode, IArchiveExtractCallback *extractCallback)
+{
+  COM_TRY_BEGIN
+  if (numItems == 0)
+    return S_OK;
+  if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
+    return E_INVALIDARG;
+
+  if (_packSize_Defined)
+    extractCallback->SetTotal(_packSize);
+
+  CMyComPtr<ISequentialOutStream> realOutStream;
+  Int32 askMode = testMode ?
+      NExtract::NAskMode::kTest :
+      NExtract::NAskMode::kExtract;
+  RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
+  if (!testMode && !realOutStream)
+    return S_OK;
+
+  extractCallback->PrepareOperation(askMode);
+
+  Int32 opRes;
+
+  {
+
+  NCompress::NLZ4::CDecoder *decoderSpec = new NCompress::NLZ4::CDecoder;
+  CMyComPtr<ICompressCoder> decoder = decoderSpec;
+  decoderSpec->SetInStream(_seqStream);
+
+  CDummyOutStream *outStreamSpec = new CDummyOutStream;
+  CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
+  outStreamSpec->SetStream(realOutStream);
+  outStreamSpec->Init();
+
+  realOutStream.Release();
+
+  CLocalProgress *lps = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> progress = lps;
+  lps->Init(extractCallback, true);
+
+  UInt64 packSize = 0;
+  UInt64 unpackedSize = 0;
+
+  HRESULT result = S_OK;
+
+  for (;;)
+  {
+    lps->InSize = packSize;
+    lps->OutSize = unpackedSize;
+
+    RINOK(lps->SetCur());
+    result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
+    UInt64 streamSize = decoderSpec->GetInputProcessedSize();
+
+    if (result != S_FALSE && result != S_OK)
+      return result;
+
+    if (unpackedSize == 0)
+      break;
+
+    if (streamSize == packSize)
+    {
+      // no new bytes in input stream, So it's good end of archive.
+      result = S_OK;
+      break;
+    }
+
+    if (packSize > streamSize)
+      return E_FAIL;
+
+    if (result != S_OK)
+      break;
+  }
+
+  decoderSpec->ReleaseInStream();
+  outStream.Release();
+
+  if (!_isArc)
+    opRes = NExtract::NOperationResult::kIsNotArc;
+  else if (_needMoreInput)
+    opRes = NExtract::NOperationResult::kUnexpectedEnd;
+  else if (_dataAfterEnd)
+    opRes = NExtract::NOperationResult::kDataAfterEnd;
+  else if (result == S_FALSE)
+    opRes = NExtract::NOperationResult::kDataError;
+  else if (result == S_OK)
+    opRes = NExtract::NOperationResult::kOK;
+  else
+    return result;
+
+  }
+
+  return extractCallback->SetOperationResult(opRes);
+
+  COM_TRY_END
+}
+
+static HRESULT UpdateArchive(
+    UInt64 unpackSize,
+    ISequentialOutStream *outStream,
+    const CProps &props,
+    IArchiveUpdateCallback *updateCallback)
+{
+  RINOK(updateCallback->SetTotal(unpackSize));
+  CMyComPtr<ISequentialInStream> fileInStream;
+  RINOK(updateCallback->GetStream(0, &fileInStream));
+  CLocalProgress *localProgressSpec = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
+  localProgressSpec->Init(updateCallback, true);
+  NCompress::NLZ4::CEncoder *encoderSpec = new NCompress::NLZ4::CEncoder;
+  CMyComPtr<ICompressCoder> encoder = encoderSpec;
+  RINOK(props.SetCoderProps(encoderSpec, NULL));
+  RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));
+  return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
+}
+
+STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
+{
+  *type = NFileTimeType::kUnix;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
+    IArchiveUpdateCallback *updateCallback)
+{
+  COM_TRY_BEGIN
+
+  if (numItems != 1)
+    return E_INVALIDARG;
+
+  Int32 newData, newProps;
+  UInt32 indexInArchive;
+  if (!updateCallback)
+    return E_FAIL;
+  RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive));
+ 
+  if ((newProps))
+  {
+    {
+      NCOM::CPropVariant prop;
+      RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop));
+      if (prop.vt != VT_EMPTY)
+        if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
+          return E_INVALIDARG;
+    }
+  }
+  
+  if ((newData))
+  {
+    UInt64 size;
+    {
+      NCOM::CPropVariant prop;
+      RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
+      if (prop.vt != VT_UI8)
+        return E_INVALIDARG;
+      size = prop.uhVal.QuadPart;
+    }
+    return UpdateArchive(size, outStream, _props, updateCallback);
+  }
+
+  if (indexInArchive != 0)
+    return E_INVALIDARG;
+
+  CLocalProgress *lps = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> progress = lps;
+  lps->Init(updateCallback, true);
+
+  CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
+  updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
+  if (opCallback)
+  {
+    RINOK(opCallback->ReportOperation(
+        NEventIndexType::kInArcIndex, 0,
+        NUpdateNotifyOp::kReplicate))
+  }
+
+  if (_stream)
+    RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
+
+  return NCompress::CopyStream(_stream, outStream, progress);
+
+  COM_TRY_END
+}
+
+STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
+{
+  return _props.SetProperties(names, values, numProps);
+}
+
+static const Byte k_Signature[] = "0x184D2204";
+
+REGISTER_ARC_IO(
+  "lz4", "lz4 tlz4", "* .tar", 0x0f,
+  k_Signature,
+  0,
+  NArcInfoFlags::kKeepName,
+  IsArc_lz4)
+
+}}

CPP/7zip/Archive/Lz5Handler.cpp

@@ -0,0 +1,366 @@
+// ZstdHandler.cpp
+
+#include "StdAfx.h"
+
+#include "../../../C/CpuArch.h"
+#include "../../Common/ComTry.h"
+#include "../../Common/Defs.h"
+
+#include "../Common/ProgressUtils.h"
+#include "../Common/RegisterArc.h"
+#include "../Common/StreamUtils.h"
+
+#include "../Compress/Lz5Decoder.h"
+#include "../Compress/Lz5Encoder.h"
+#include "../Compress/CopyCoder.h"
+
+#include "Common/DummyOutStream.h"
+#include "Common/HandlerOut.h"
+
+using namespace NWindows;
+
+namespace NArchive {
+namespace NLZ5 {
+
+class CHandler:
+  public IInArchive,
+  public IArchiveOpenSeq,
+  public IOutArchive,
+  public ISetProperties,
+  public CMyUnknownImp
+{
+  CMyComPtr<IInStream> _stream;
+  CMyComPtr<ISequentialInStream> _seqStream;
+
+  bool _isArc;
+  bool _dataAfterEnd;
+  bool _needMoreInput;
+
+  bool _packSize_Defined;
+  bool _unpackSize_Defined;
+
+  UInt64 _packSize;
+  UInt64 _unpackSize;
+  UInt64 _numStreams;
+  UInt64 _numBlocks;
+
+  CSingleMethodProps _props;
+
+public:
+  MY_UNKNOWN_IMP4(
+      IInArchive,
+      IArchiveOpenSeq,
+      IOutArchive,
+      ISetProperties)
+  INTERFACE_IInArchive(;)
+  INTERFACE_IOutArchive(;)
+  STDMETHOD(OpenSeq)(ISequentialInStream *stream);
+  STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
+
+  CHandler() { }
+};
+
+static const Byte kProps[] =
+{
+  kpidSize,
+  kpidPackSize
+};
+
+static const Byte kArcProps[] =
+{
+  kpidNumStreams,
+  kpidNumBlocks
+};
+
+IMP_IInArchive_Props
+IMP_IInArchive_ArcProps
+
+STDMETHODIMP CHandler::GetArchiveProperty(PROPID /*propID*/, PROPVARIANT * /*value*/)
+{
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
+{
+  *numItems = 1;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
+{
+  NCOM::CPropVariant prop;
+  switch (propID)
+  {
+    case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
+    case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
+  }
+  prop.Detach(value);
+  return S_OK;
+}
+
+static const unsigned kSignatureCheckSize = 4;
+
+API_FUNC_static_IsArc IsArc_lz5(const Byte *p, size_t size)
+{
+  if (size < 4)
+    return k_IsArc_Res_NEED_MORE;
+
+  UInt32 magic = GetUi32(p);
+
+  // skippable frames
+  if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) {
+    if (size < 16)
+      return k_IsArc_Res_NEED_MORE;
+    magic = GetUi32(p+12);
+  }
+
+  // Lz5 Magic
+  if (magic == 0x184D2205)
+    return k_IsArc_Res_YES;
+
+  return k_IsArc_Res_NO;
+}
+}
+
+STDMETHODIMP CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *)
+{
+  COM_TRY_BEGIN
+  Close();
+  {
+    Byte buf[kSignatureCheckSize];
+    RINOK(ReadStream_FALSE(stream, buf, kSignatureCheckSize));
+    if (IsArc_lz5(buf, kSignatureCheckSize) == k_IsArc_Res_NO)
+      return S_FALSE;
+    _isArc = true;
+    _stream = stream;
+    _seqStream = stream;
+    RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
+  }
+  return S_OK;
+  COM_TRY_END
+}
+
+STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
+{
+  Close();
+  _isArc = true;
+  _seqStream = stream;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::Close()
+{
+  _isArc = false;
+  _dataAfterEnd = false;
+  _needMoreInput = false;
+
+  _packSize_Defined = false;
+  _unpackSize_Defined = false;
+
+  _packSize = 0;
+
+  _seqStream.Release();
+  _stream.Release();
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
+    Int32 testMode, IArchiveExtractCallback *extractCallback)
+{
+  COM_TRY_BEGIN
+  if (numItems == 0)
+    return S_OK;
+  if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
+    return E_INVALIDARG;
+
+  if (_packSize_Defined)
+    extractCallback->SetTotal(_packSize);
+
+  CMyComPtr<ISequentialOutStream> realOutStream;
+  Int32 askMode = testMode ?
+      NExtract::NAskMode::kTest :
+      NExtract::NAskMode::kExtract;
+  RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
+  if (!testMode && !realOutStream)
+    return S_OK;
+
+  extractCallback->PrepareOperation(askMode);
+
+  Int32 opRes;
+
+  {
+
+  NCompress::NLZ5::CDecoder *decoderSpec = new NCompress::NLZ5::CDecoder;
+  CMyComPtr<ICompressCoder> decoder = decoderSpec;
+  decoderSpec->SetInStream(_seqStream);
+
+  CDummyOutStream *outStreamSpec = new CDummyOutStream;
+  CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
+  outStreamSpec->SetStream(realOutStream);
+  outStreamSpec->Init();
+
+  realOutStream.Release();
+
+  CLocalProgress *lps = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> progress = lps;
+  lps->Init(extractCallback, true);
+
+  UInt64 packSize = 0;
+  UInt64 unpackedSize = 0;
+
+  HRESULT result = S_OK;
+
+  for (;;)
+  {
+    lps->InSize = packSize;
+    lps->OutSize = unpackedSize;
+
+    RINOK(lps->SetCur());
+    result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
+    UInt64 streamSize = decoderSpec->GetInputProcessedSize();
+
+    if (result != S_FALSE && result != S_OK)
+      return result;
+
+    if (unpackedSize == 0)
+      break;
+
+    if (streamSize == packSize)
+    {
+      // no new bytes in input stream, So it's good end of archive.
+      result = S_OK;
+      break;
+    }
+
+    if (packSize > streamSize)
+      return E_FAIL;
+
+    if (result != S_OK)
+      break;
+  }
+
+  decoderSpec->ReleaseInStream();
+  outStream.Release();
+
+  if (!_isArc)
+    opRes = NExtract::NOperationResult::kIsNotArc;
+  else if (_needMoreInput)
+    opRes = NExtract::NOperationResult::kUnexpectedEnd;
+  else if (_dataAfterEnd)
+    opRes = NExtract::NOperationResult::kDataAfterEnd;
+  else if (result == S_FALSE)
+    opRes = NExtract::NOperationResult::kDataError;
+  else if (result == S_OK)
+    opRes = NExtract::NOperationResult::kOK;
+  else
+    return result;
+
+  }
+
+  return extractCallback->SetOperationResult(opRes);
+
+  COM_TRY_END
+}
+
+static HRESULT UpdateArchive(
+    UInt64 unpackSize,
+    ISequentialOutStream *outStream,
+    const CProps &props,
+    IArchiveUpdateCallback *updateCallback)
+{
+  RINOK(updateCallback->SetTotal(unpackSize));
+  CMyComPtr<ISequentialInStream> fileInStream;
+  RINOK(updateCallback->GetStream(0, &fileInStream));
+  CLocalProgress *localProgressSpec = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
+  localProgressSpec->Init(updateCallback, true);
+  NCompress::NLZ5::CEncoder *encoderSpec = new NCompress::NLZ5::CEncoder;
+  CMyComPtr<ICompressCoder> encoder = encoderSpec;
+  RINOK(props.SetCoderProps(encoderSpec, NULL));
+  RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));
+  return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
+}
+
+STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
+{
+  *type = NFileTimeType::kUnix;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
+    IArchiveUpdateCallback *updateCallback)
+{
+  COM_TRY_BEGIN
+
+  if (numItems != 1)
+    return E_INVALIDARG;
+
+  Int32 newData, newProps;
+  UInt32 indexInArchive;
+  if (!updateCallback)
+    return E_FAIL;
+  RINOK(updateCallback->GetUpdateItemInfo(0, &newData, &newProps, &indexInArchive));
+ 
+  if ((newProps))
+  {
+    {
+      NCOM::CPropVariant prop;
+      RINOK(updateCallback->GetProperty(0, kpidIsDir, &prop));
+      if (prop.vt != VT_EMPTY)
+        if (prop.vt != VT_BOOL || prop.boolVal != VARIANT_FALSE)
+          return E_INVALIDARG;
+    }
+  }
+  
+  if ((newData))
+  {
+    UInt64 size;
+    {
+      NCOM::CPropVariant prop;
+      RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
+      if (prop.vt != VT_UI8)
+        return E_INVALIDARG;
+      size = prop.uhVal.QuadPart;
+    }
+    return UpdateArchive(size, outStream, _props, updateCallback);
+  }
+
+  if (indexInArchive != 0)
+    return E_INVALIDARG;
+
+  CLocalProgress *lps = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> progress = lps;
+  lps->Init(updateCallback, true);
+
+  CMyComPtr<IArchiveUpdateCallbackFile> opCallback;
+  updateCallback->QueryInterface(IID_IArchiveUpdateCallbackFile, (void **)&opCallback);
+  if (opCallback)
+  {
+    RINOK(opCallback->ReportOperation(
+        NEventIndexType::kInArcIndex, 0,
+        NUpdateNotifyOp::kReplicate))
+  }
+
+  if (_stream)
+    RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
+
+  return NCompress::CopyStream(_stream, outStream, progress);
+
+  COM_TRY_END
+}
+
+STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
+{
+  return _props.SetProperties(names, values, numProps);
+}
+
+static const Byte k_Signature[] = "0x184D2205";
+
+REGISTER_ARC_IO(
+  "lz5", "lz5 tlz5", "* .tar", 0x10,
+  k_Signature,
+  0,
+  NArcInfoFlags::kKeepName,
+  IsArc_lz5)
+
+}}

CPP/7zip/Archive/LzHandler.cpp

@@ -0,0 +1,460 @@
+// LzHandler.cpp
+
+#include "StdAfx.h"
+
+#include "../../../C/CpuArch.h"
+
+#include "../../Common/ComTry.h"
+#include "../../Common/IntToString.h"
+
+#include "../../Windows/PropVariant.h"
+
+#include "../Common/ProgressUtils.h"
+#include "../Common/RegisterArc.h"
+#include "../Common/StreamUtils.h"
+
+#include "../Compress/LzmaDecoder.h"
+
+#include "Common/OutStreamWithCRC.h"
+
+using namespace NWindows;
+
+namespace NArchive {
+namespace NLz {
+
+static const Byte kProps[] =
+{
+  kpidSize,
+  kpidPackSize,
+};
+
+static const Byte kArcProps[] =
+{
+  kpidNumStreams
+};
+
+static const Byte k_Signature[5] = { 'L', 'Z', 'I', 'P', 1 };
+enum { k_SignatureSize = 5 };
+
+struct CHeader
+{
+  Byte data[6];		// 0-3 magic bytes, 4 version, 5 coded_dict_size
+  enum { size = 6 };
+  enum { min_dictionary_size = 1 << 12, max_dictionary_size = 1 << 29 };
+  unsigned DicSize;
+  Byte LzmaProps[5];
+
+  bool Parse();
+};
+
+struct CTrailer
+{
+  Byte data[20];	//  0-3  CRC32 of the uncompressed data
+			//  4-11 size of the uncompressed data
+			// 12-19 member size including header and trailer
+  enum { size = 20 };
+
+  unsigned data_crc() const
+    {
+    unsigned tmp = 0;
+    for( int i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; }
+    return tmp;
+    }
+
+  UInt64 data_size() const
+    {
+    UInt64 tmp = 0;
+    for( int i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; }
+    return tmp;
+    }
+
+  UInt64 member_size() const
+    {
+    UInt64 tmp = 0;
+    for( int i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; }
+    return tmp;
+    }
+};
+
+class CDecoder
+{
+  CMyComPtr<ICompressCoder> _lzmaDecoder;
+public:
+  NCompress::NLzma::CDecoder *_lzmaDecoderSpec;
+
+  ~CDecoder();
+  HRESULT Create(ISequentialInStream *inStream);
+
+  HRESULT Code(const CHeader &header, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
+
+  UInt64 GetInputProcessedSize() const { return _lzmaDecoderSpec->GetInputProcessedSize(); }
+
+  void ReleaseInStream() { if (_lzmaDecoder) _lzmaDecoderSpec->ReleaseInStream(); }
+
+  HRESULT ReadInput(Byte *data, UInt32 size, UInt32 *processedSize)
+    { return _lzmaDecoderSpec->ReadFromInputStream(data, size, processedSize); }
+};
+
+HRESULT CDecoder::Create(ISequentialInStream *inStream)
+{
+  if (!_lzmaDecoder)
+  {
+    _lzmaDecoderSpec = new NCompress::NLzma::CDecoder;
+    _lzmaDecoderSpec->FinishStream = true;
+    _lzmaDecoder = _lzmaDecoderSpec;
+  }
+
+  return _lzmaDecoderSpec->SetInStream(inStream);
+}
+
+CDecoder::~CDecoder()
+{
+  ReleaseInStream();
+}
+
+HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream,
+    ICompressProgressInfo *progress)
+{
+  {
+    CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties;
+    _lzmaDecoder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecoderProperties);
+    if (!setDecoderProperties)
+      return E_NOTIMPL;
+    RINOK(setDecoderProperties->SetDecoderProperties2(header.LzmaProps, 5));
+  }
+
+  RINOK(_lzmaDecoderSpec->CodeResume(outStream, NULL, progress));
+
+  return S_OK;
+}
+
+
+class CHandler:
+  public IInArchive,
+  public IArchiveOpenSeq,
+  public CMyUnknownImp
+{
+  CHeader _header;
+  CMyComPtr<IInStream> _stream;
+  CMyComPtr<ISequentialInStream> _seqStream;
+
+  bool _isArc;
+  bool _needSeekToStart;
+  bool _dataAfterEnd;
+  bool _needMoreInput;
+
+  bool _packSize_Defined;
+  bool _unpackSize_Defined;
+  bool _numStreams_Defined;
+
+  bool _dataError;
+
+  UInt64 _packSize;
+  UInt64 _unpackSize;
+  UInt64 _numStreams;
+
+public:
+  MY_UNKNOWN_IMP2(IInArchive, IArchiveOpenSeq)
+
+  INTERFACE_IInArchive(;)
+  STDMETHOD(OpenSeq)(ISequentialInStream *stream);
+
+  CHandler() { }
+};
+
+IMP_IInArchive_Props
+IMP_IInArchive_ArcProps
+
+STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
+{
+  NCOM::CPropVariant prop;
+  switch (propID)
+  {
+    case kpidPhySize: if (_packSize_Defined) prop = _packSize; break;
+    case kpidUnpackSize: if (_unpackSize_Defined) prop = _unpackSize; break;
+    case kpidNumStreams: if (_numStreams_Defined) prop = _numStreams; break;
+    case kpidErrorFlags:
+    {
+      UInt32 v = 0;
+      if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;;
+      if (_needMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd;
+      if (_dataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd;
+      if (_dataError) v |= kpv_ErrorFlags_DataError;
+      prop = v;
+    }
+  }
+  prop.Detach(value);
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetNumberOfItems(UInt32 *numItems)
+{
+  *numItems = 1;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetProperty(UInt32 /* index */, PROPID propID, PROPVARIANT *value)
+{
+  NCOM::CPropVariant prop;
+  switch (propID)
+  {
+    case kpidSize: if (_unpackSize_Defined) prop = _unpackSize; break;
+    case kpidPackSize: if (_packSize_Defined) prop = _packSize; break;
+  }
+  prop.Detach(value);
+  return S_OK;
+}
+
+API_FUNC_static_IsArc IsArc_Lz(const Byte *p, size_t size)
+{
+  if (size < k_SignatureSize)
+    return k_IsArc_Res_NEED_MORE;
+  for( int i = 0; i < k_SignatureSize; ++i )
+    if( p[i] != k_Signature[i] )
+      return k_IsArc_Res_NO;
+  return k_IsArc_Res_YES;
+}
+}
+
+bool CHeader::Parse()
+{
+  if (IsArc_Lz(data, k_SignatureSize) == k_IsArc_Res_NO)
+    return false;
+  DicSize = ( 1 << ( data[5] & 0x1F ) );
+  if( DicSize > min_dictionary_size )
+    DicSize -= ( DicSize / 16 ) * ( ( data[5] >> 5 ) & 7 );
+  LzmaProps[0] = 93;			/* (45 * 2) + (9 * 0) + 3 */
+  unsigned ds = DicSize;
+  for( int i = 1; i <= 4; ++i ) { LzmaProps[i] = ds & 0xFF; ds >>= 8; }
+  return (DicSize >= min_dictionary_size && DicSize <= max_dictionary_size);
+}
+
+STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *)
+{
+  Close();
+
+  RINOK(ReadStream_FALSE(inStream, _header.data, CHeader::size));
+
+  if (!_header.Parse())
+    return S_FALSE;
+
+  RINOK(inStream->Seek(0, STREAM_SEEK_END, &_packSize));
+  if (_packSize < 36)
+    return S_FALSE;
+  _isArc = true;
+  _stream = inStream;
+  _seqStream = inStream;
+  _needSeekToStart = true;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::OpenSeq(ISequentialInStream *stream)
+{
+  Close();
+  _isArc = true;
+  _seqStream = stream;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::Close()
+{
+  _isArc = false;
+  _packSize_Defined = false;
+  _unpackSize_Defined = false;
+  _numStreams_Defined = false;
+
+  _dataAfterEnd = false;
+  _needMoreInput = false;
+  _dataError = false;
+
+  _packSize = 0;
+
+  _needSeekToStart = false;
+
+  _stream.Release();
+  _seqStream.Release();
+   return S_OK;
+}
+
+class CCompressProgressInfoImp:
+  public ICompressProgressInfo,
+  public CMyUnknownImp
+{
+  CMyComPtr<IArchiveOpenCallback> Callback;
+public:
+  UInt64 Offset;
+
+  MY_UNKNOWN_IMP1(ICompressProgressInfo)
+  STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize);
+  void Init(IArchiveOpenCallback *callback) { Callback = callback; }
+};
+
+STDMETHODIMP CCompressProgressInfoImp::SetRatioInfo(const UInt64 *inSize, const UInt64 * /* outSize */)
+{
+  if (Callback)
+  {
+    UInt64 files = 0;
+    UInt64 value = Offset + *inSize;
+    return Callback->SetCompleted(&files, &value);
+  }
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
+    Int32 testMode, IArchiveExtractCallback *extractCallback)
+{
+  COM_TRY_BEGIN
+  if (numItems == 0)
+    return S_OK;
+  if (numItems != (UInt32)(Int32)-1 && (numItems != 1 || indices[0] != 0))
+    return E_INVALIDARG;
+
+  if (_packSize_Defined)
+    extractCallback->SetTotal(_packSize);
+
+  CMyComPtr<ISequentialOutStream> realOutStream;
+  Int32 askMode = testMode ?
+      NExtract::NAskMode::kTest :
+      NExtract::NAskMode::kExtract;
+  RINOK(extractCallback->GetStream(0, &realOutStream, askMode));
+  if (!testMode && !realOutStream)
+    return S_OK;
+
+  extractCallback->PrepareOperation(askMode);
+
+  COutStreamWithCRC *outStreamSpec = new COutStreamWithCRC;
+  CMyComPtr<ISequentialOutStream> outStream(outStreamSpec);
+  outStreamSpec->SetStream(realOutStream);
+  outStreamSpec->Init();
+  realOutStream.Release();
+
+  CLocalProgress *lps = new CLocalProgress;
+  CMyComPtr<ICompressProgressInfo> progress = lps;
+  lps->Init(extractCallback, true);
+
+  if (_needSeekToStart)
+  {
+    if (!_stream)
+      return E_FAIL;
+    RINOK(_stream->Seek(0, STREAM_SEEK_SET, NULL));
+  }
+  else
+    _needSeekToStart = true;
+
+  CDecoder decoder;
+  HRESULT result = decoder.Create(_seqStream);
+  RINOK(result);
+
+  bool firstItem = true;
+
+  UInt64 packSize = 0;
+  UInt64 unpackSize = 0;
+  UInt64 numStreams = 0;
+
+  bool crcError = false;
+  bool dataAfterEnd = false;
+
+  for (;;)
+  {
+    lps->InSize = packSize;
+    lps->OutSize = unpackSize;
+    RINOK(lps->SetCur());
+
+    CHeader st;
+    UInt32 processed;
+    RINOK(decoder.ReadInput(st.data, CHeader::size, &processed));
+    if (processed != CHeader::size)
+    {
+      if (processed != 0)
+        dataAfterEnd = true;
+      break;
+    }
+
+    if (!st.Parse())
+    {
+      dataAfterEnd = true;
+      break;
+    }
+    numStreams++;
+    firstItem = false;
+    outStreamSpec->InitCRC();
+
+    result = decoder.Code(st, outStream, progress);
+
+    UInt64 member_size = decoder.GetInputProcessedSize() - packSize;
+    packSize += member_size;
+    UInt64 data_size = outStreamSpec->GetSize() - unpackSize;
+    unpackSize += data_size;
+
+    if (result == S_OK)
+    {
+      CTrailer trailer;
+      RINOK(decoder.ReadInput(trailer.data, CTrailer::size, &processed));
+      packSize += processed; member_size += processed;
+      if (processed != CTrailer::size ||
+          trailer.data_crc() != outStreamSpec->GetCRC() ||
+          trailer.data_size() != data_size ||
+          trailer.member_size() != member_size)
+      {
+        crcError = true;
+        result = S_FALSE;
+        break;
+      }
+    }
+    if (result == S_FALSE)
+      break;
+    RINOK(result);
+  }
+
+  if (firstItem)
+  {
+    _isArc = false;
+    result = S_FALSE;
+  }
+  else if (result == S_OK || result == S_FALSE)
+  {
+    if (dataAfterEnd)
+      _dataAfterEnd = true;
+    else if (decoder._lzmaDecoderSpec->NeedMoreInput)
+      _needMoreInput = true;
+
+    _packSize = packSize;
+    _unpackSize = unpackSize;
+    _numStreams = numStreams;
+
+    _packSize_Defined = true;
+    _unpackSize_Defined = true;
+    _numStreams_Defined = true;
+  }
+
+  Int32 opResult = NExtract::NOperationResult::kOK;
+
+  if (!_isArc)
+    opResult = NExtract::NOperationResult::kIsNotArc;
+  else if (_needMoreInput)
+    opResult = NExtract::NOperationResult::kUnexpectedEnd;
+  else if (crcError)
+    opResult = NExtract::NOperationResult::kCRCError;
+  else if (_dataAfterEnd)
+    opResult = NExtract::NOperationResult::kDataAfterEnd;
+  else if (result == S_FALSE)
+    opResult = NExtract::NOperationResult::kDataError;
+  else if (result == S_OK)
+    opResult = NExtract::NOperationResult::kOK;
+  else
+    return result;
+
+  outStream.Release();
+  return extractCallback->SetOperationResult(opResult);
+
+  COM_TRY_END
+}
+
+REGISTER_ARC_I(
+  "lzip", "lz tlz", "* .tar", 0xC6,
+  k_Signature,
+  0,
+  NArcInfoFlags::kKeepName,
+  IsArc_Lz)
+
+}}

CPP/7zip/Archive/ZstdHandler.cpp

@@ -17,8 +17,6 @@
 #include "Common/DummyOutStream.h"
 #include "Common/HandlerOut.h"
 
-#include <stdio.h>
-
 using namespace NWindows;
 
 namespace NArchive {
@@ -109,6 +107,14 @@ API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size)
 
   UInt32 magic = GetUi32(p);
 
+  // skippable frames
+  if (magic >= 0x184D2A50 && magic <= 0x184D2A5F) {
+    if (size < 16)
+      return k_IsArc_Res_NEED_MORE;
+    magic = GetUi32(p+12);
+  }
+
+#ifdef ZSTD_LEGACY_SUPPORT
   // zstd 0.1
   if (magic == 0xFD2FB51E)
     return k_IsArc_Res_YES;
@@ -116,10 +122,11 @@ API_FUNC_static_IsArc IsArc_zstd(const Byte *p, size_t size)
   // zstd magic's for 0.2 .. 0.8 (aka 1.x)
   if (magic >= 0xFD2FB522 && magic <= 0xFD2FB528)
     return k_IsArc_Res_YES;
-
+#else
-  // skippable frames
+  /* only version 1.x */
-  if (magic >= 0x184D2A50 && magic <= 0x184D2A5F)
+  if (magic == 0xFD2FB528)
     return k_IsArc_Res_YES;
+#endif
 
   return k_IsArc_Res_NO;
 }
@@ -223,10 +230,6 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
     result = decoderSpec->CodeResume(outStream, &unpackedSize, progress);
     UInt64 streamSize = decoderSpec->GetInputProcessedSize();
 
-    printf("streamsize=%d packsize=%d unpackedSize=%d\n",
-      streamSize, packSize, unpackedSize);
-    fflush(stdout);
-
     if (result != S_FALSE && result != S_OK)
       return result;
 
@@ -362,7 +365,7 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR
   return _props.SetProperties(names, values, numProps);
 }
 
-static const Byte k_Signature[] = "0xFD2FB525..0xFD2FB528";
+static const Byte k_Signature[] = "0xFD2FB522..28";
 
 REGISTER_ARC_IO(
   "zstd", "zst tzstd", "* .tar", 0x0e,

CPP/7zip/Bundles/Alone/Alone.dsp

@@ -1618,6 +1618,10 @@ SOURCE=..\..\Archive\IArchive.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\Archive\LzHandler.cpp
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\Archive\LzmaHandler.cpp
 # End Source File
 # Begin Source File

CPP/7zip/Bundles/Alone/makefile

@@ -70,6 +70,7 @@ AR_OBJS = \
   $O\Bz2Handler.obj \
   $O\DeflateProps.obj \
   $O\GzHandler.obj \
+  $O\LzHandler.obj \
   $O\LzmaHandler.obj \
   $O\SplitHandler.obj \
   $O\XzHandler.obj \

CPP/7zip/Bundles/Codec_lz4/StdAfx.cpp

@@ -0,0 +1,3 @@
+// StdAfx.cpp
+
+#include "StdAfx.h"

CPP/7zip/Bundles/Codec_lz4/StdAfx.h

@@ -0,0 +1,8 @@
+// StdAfx.h
+
+#ifndef __STDAFX_H
+#define __STDAFX_H
+
+#include "../../../Common/Common.h"
+
+#endif

CPP/7zip/Bundles/Codec_lz4/makefile

@@ -0,0 +1,36 @@
+PROG = lz4.dll
+DEF_FILE = ../../Compress/Codec.def
+CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID
+
+7ZIP_COMMON_OBJS = \
+  $O\StreamUtils.obj \
+
+WIN_OBJS = \
+  $O\System.obj \
+
+COMPRESS_OBJS = \
+  $O\CodecExports.obj \
+  $O\DllExportsCompress.obj \
+
+C_OBJS = \
+  $O\Alloc.obj \
+  $O\Threads.obj \
+
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
+  $O\Lz4Decoder.obj \
+  $O\Lz4Encoder.obj \
+  $O\Lz4Register.obj \
+
+LZ4_OBJS = \
+  $O\lz4.obj \
+  $O\lz4frame.obj \
+  $O\lz4hc.obj \
+  $O\lz4xxhash.obj \
+
+ZSTDMT_OBJS = \
+  $O\lz4mt_common.obj \
+  $O\lz4mt_compress.obj \
+  $O\lz4mt_decompress.obj \
+  $O\threading.obj \
+
+!include "../../7zip.mak"

CPP/7zip/Bundles/Codec_lz4/resource.rc

@@ -0,0 +1,6 @@
+#include "../../../../C/7zVersionTr.h"
+#include "../../../../C/7zVersion.rc"
+
+MY_VERSION_INFO_DLL("7z LZ4 Plugin", "lz4")
+
+101  ICON  "../../Archive/Icons/7z.ico"

CPP/7zip/Bundles/Codec_lz5/StdAfx.cpp

@@ -0,0 +1,3 @@
+// StdAfx.cpp
+
+#include "StdAfx.h"

CPP/7zip/Bundles/Codec_lz5/StdAfx.h

@@ -0,0 +1,8 @@
+// StdAfx.h
+
+#ifndef __STDAFX_H
+#define __STDAFX_H
+
+#include "../../../Common/Common.h"
+
+#endif

CPP/7zip/Bundles/Codec_lz5/makefile

@@ -0,0 +1,36 @@
+PROG = lz5.dll
+DEF_FILE = ../../Compress/Codec.def
+CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID
+
+7ZIP_COMMON_OBJS = \
+  $O\StreamUtils.obj \
+
+WIN_OBJS = \
+  $O\System.obj \
+
+COMPRESS_OBJS = \
+  $O\CodecExports.obj \
+  $O\DllExportsCompress.obj \
+
+C_OBJS = \
+  $O\Alloc.obj \
+  $O\Threads.obj \
+
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
+  $O\Lz5Decoder.obj \
+  $O\Lz5Encoder.obj \
+  $O\Lz5Register.obj \
+
+LZ5_OBJS = \
+  $O\lz5.obj \
+  $O\lz5frame.obj \
+  $O\lz5hc.obj \
+  $O\lz5xxhash.obj \
+
+ZSTDMT_OBJS = \
+  $O\lz5mt_common.obj \
+  $O\lz5mt_compress.obj \
+  $O\lz5mt_decompress.obj \
+  $O\threading.obj \
+
+!include "../../7zip.mak"

CPP/7zip/Bundles/Codec_lz5/resource.rc

@@ -0,0 +1,6 @@
+#include "../../../../C/7zVersionTr.h"
+#include "../../../../C/7zVersion.rc"
+
+MY_VERSION_INFO_DLL("7z LZ5 Plugin", "lz5")
+
+101  ICON  "../../Archive/Icons/7z.ico"

CPP/7zip/Bundles/Codec_zstd/makefile

@@ -5,17 +5,38 @@ CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID
 7ZIP_COMMON_OBJS = \
   $O\StreamUtils.obj \
 
+WIN_OBJS = \
+  $O\System.obj \
+
 COMPRESS_OBJS = \
   $O\CodecExports.obj \
   $O\DllExportsCompress.obj \
 
 C_OBJS = \
   $O\Alloc.obj \
+  $O\Threads.obj \
 
-!include "../../zstd.mak"
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
-ZSTD_OBJS = $(ZSTD_OBJS) \
+  $O\ZstdDecoder.obj \
+  $O\ZstdEncoder.obj \
+  $O\ZstdRegister.obj \
+
+ZSTD_OBJS = \
+  $O\entropy_common.obj \
+  $O\fse_decompress.obj \
+  $O\huf_decompress.obj \
+  $O\zstd_common.obj \
+  $O\zstd_decompress.obj \
+  $O\xxhash.obj \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
   $O\zstd_compress.obj \
+  $O\error_private.obj \
+
+ZSTDMT_OBJS = \
+  $O\threading.obj \
+  $O\zstdmt_common.obj \
+  $O\zstdmt_compress.obj \
+  $O\zstdmt_decompress.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Codec_zstdF/makefile

@@ -5,18 +5,33 @@ CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_LEGACY_SUPPORT
 7ZIP_COMMON_OBJS = \
   $O\StreamUtils.obj \
 
+WIN_OBJS = \
+  $O\System.obj \
+
 COMPRESS_OBJS = \
   $O\CodecExports.obj \
   $O\DllExportsCompress.obj \
 
 C_OBJS = \
   $O\Alloc.obj \
+  $O\Threads.obj \
 
-!include "../../zstd.mak"
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
-ZSTD_OBJS = $(ZSTD_OBJS) \
+  $O\ZstdDecoder.obj \
+  $O\ZstdEncoder.obj \
+  $O\ZstdRegister.obj \
+
+ZSTD_OBJS = \
+  $O\entropy_common.obj \
+  $O\fse_decompress.obj \
+  $O\huf_decompress.obj \
+  $O\zstd_common.obj \
+  $O\zstd_decompress.obj \
+  $O\xxhash.obj \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
   $O\zstd_compress.obj \
+  $O\error_private.obj \
   $O\zstd_v01.obj \
   $O\zstd_v02.obj \
   $O\zstd_v03.obj \
@@ -25,4 +40,10 @@ ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\zstd_v06.obj \
   $O\zstd_v07.obj \
 
+ZSTDMT_OBJS = \
+  $O\threading.obj \
+  $O\zstdmt_common.obj \
+  $O\zstdmt_compress.obj \
+  $O\zstdmt_decompress.obj \
+
 !include "../../7zip.mak"

CPP/7zip/Bundles/Fm/resource.rc

@@ -3,5 +3,5 @@
 
 STRINGTABLE
 BEGIN
-  100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs"
+  100 "7z zip rar 001 cab iso xz txz lzma tar cpio bz2 bzip2 tbz2 tbz gz gzip tgz tpz z taz lz tlz lz4 tlz4 lz5 tlz5 lzh lha rpm deb arj vhd wim swm fat ntfs dmg hfs xar squashfs zst"
 END

CPP/7zip/Bundles/Format7z/makefile

@@ -136,10 +136,28 @@ C_OBJS = \
 
 !include "../../Aes.mak"
 !include "../../Crc.mak"
-!include "../../zstd.mak"
+
-ZSTD_OBJS = $(ZSTD_OBJS) \
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
+  $O\ZstdDecoder.obj \
+  $O\ZstdEncoder.obj \
+  $O\ZstdRegister.obj \
+
+ZSTD_OBJS = \
+  $O\entropy_common.obj \
+  $O\fse_decompress.obj \
+  $O\huf_decompress.obj \
+  $O\zstd_common.obj \
+  $O\zstd_decompress.obj \
+  $O\xxhash.obj \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
   $O\zstd_compress.obj \
+  $O\error_private.obj \
+
+ZSTDMT_OBJS = \
+  $O\threading.obj \
+  $O\zstdmt_common.obj \
+  $O\zstdmt_compress.obj \
+  $O\zstdmt_decompress.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zF/Arc.mak

@@ -69,6 +69,9 @@ AR_OBJS = \
   $O\HandlerCont.obj \
   $O\HfsHandler.obj \
   $O\IhexHandler.obj \
+  $O\LzHandler.obj \
+  $O\Lz4Handler.obj \
+  $O\Lz5Handler.obj \
   $O\LzhHandler.obj \
   $O\LzmaHandler.obj \
   $O\MachoHandler.obj \
@@ -104,7 +107,6 @@ AR_COMMON_OBJS = \
   $O\HandlerOut.obj \
   $O\ParseProperties.obj \
 
-
 7Z_OBJS = \
   $O\7zCompressionMode.obj \
   $O\7zDecode.obj \
@@ -242,7 +244,6 @@ CRYPTO_OBJS = \
   $O\ZipCrypto.obj \
   $O\ZipStrong.obj \
 
-
 C_OBJS = \
   $O\7zBuf2.obj \
   $O\7zStream.obj \

CPP/7zip/Bundles/Format7zF/makefile

@@ -16,12 +16,39 @@ AR_OBJS = $(AR_OBJS) \
   $O\ArchiveExports.obj \
   $O\DllExports2.obj \
 
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
+  $O\Lz4Decoder.obj \
+  $O\Lz4Encoder.obj \
+  $O\Lz4Register.obj \
+  $O\Lz5Decoder.obj \
+  $O\Lz5Encoder.obj \
+  $O\Lz5Register.obj \
+  $O\ZstdDecoder.obj \
+  $O\ZstdEncoder.obj \
+  $O\ZstdRegister.obj \
 
-!include "../../zstd.mak"
+LZ4_OBJS = \
-ZSTD_OBJS = $(ZSTD_OBJS) \
+  $O\lz4.obj \
+  $O\lz4frame.obj \
+  $O\lz4hc.obj \
+  $O\lz4xxhash.obj \
+
+LZ5_OBJS = \
+  $O\lz5.obj \
+  $O\lz5frame.obj \
+  $O\lz5hc.obj \
+
+ZSTD_OBJS = \
+  $O\entropy_common.obj \
+  $O\fse_decompress.obj \
+  $O\huf_decompress.obj \
+  $O\zstd_common.obj \
+  $O\zstd_decompress.obj \
+  $O\xxhash.obj \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
   $O\zstd_compress.obj \
+  $O\error_private.obj \
   $O\zstd_v01.obj \
   $O\zstd_v02.obj \
   $O\zstd_v03.obj \
@@ -30,4 +57,16 @@ ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\zstd_v06.obj \
   $O\zstd_v07.obj \
 
+ZSTDMT_OBJS = \
+  $O\lz5mt_common.obj \
+  $O\lz5mt_compress.obj \
+  $O\lz5mt_decompress.obj \
+  $O\lz4mt_common.obj \
+  $O\lz4mt_compress.obj \
+  $O\lz4mt_decompress.obj \
+  $O\threading.obj \
+  $O\zstdmt_common.obj \
+  $O\zstdmt_compress.obj \
+  $O\zstdmt_decompress.obj \
+
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zF/resource.rc

@@ -29,9 +29,11 @@ MY_VERSION_INFO_DLL("7z Plugin", "7z")
 23  ICON  "../../Archive/Icons/xz.ico"
 24  ICON  "../../Archive/Icons/squashfs.ico"
 25  ICON  "../../Archive/Icons/zst.ico"
+26  ICON  "../../Archive/Icons/lz4.ico"
+27  ICON  "../../Archive/Icons/lz5.ico"
 
 
 STRINGTABLE
 BEGIN
-  100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25"
+  100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lz:16 tlz:16 lz4:26 lz5:27 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25"
 END

CPP/7zip/Bundles/SFXCon/makefile

@@ -41,6 +41,7 @@ WIN_OBJS = \
   $O\PropVariant.obj \
   $O\PropVariantConv.obj \
   $O\Synchronization.obj \
+  $O\System.obj \
 
 7ZIP_COMMON_OBJS = \
   $O\CreateCoder.obj \
@@ -126,5 +127,22 @@ C_OBJS = \
 !include "../../Aes.mak"
 !include "../../Crc.mak"
 
-!include "../../zstd.mak"
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
+  $O\ZstdDecoder.obj \
+  $O\ZstdRegister.obj \
+
+ZSTD_OBJS = \
+  $O\entropy_common.obj \
+  $O\fse_decompress.obj \
+  $O\huf_decompress.obj \
+  $O\zstd_common.obj \
+  $O\zstd_decompress.obj \
+  $O\xxhash.obj \
+  $O\error_private.obj \
+
+ZSTDMT_OBJS = \
+  $O\threading.obj \
+  $O\zstdmt_common.obj \
+  $O\zstdmt_decompress.obj \
+
 !include "../../7zip.mak"

CPP/7zip/Bundles/SFXSetup/makefile

@@ -106,6 +106,4 @@ C_OBJS = \
   $O\Threads.obj \
 
 !include "../../Crc.mak"
-
-!include "../../zstd.mak"
 !include "../../7zip.mak"

CPP/7zip/Bundles/SFXWin/makefile

@@ -40,6 +40,7 @@ WIN_OBJS = \
   $O\PropVariantConv.obj \
   $O\ResourceString.obj \
   $O\Shell.obj \
+  $O\System.obj \
   $O\Synchronization.obj \
   $O\Window.obj \
 
@@ -144,5 +145,22 @@ C_OBJS = \
 !include "../../Aes.mak"
 !include "../../Crc.mak"
 
-!include "../../zstd.mak"
+COMPRESS_OBJS = $(COMPRESS_OBJS) \
+  $O\ZstdDecoder.obj \
+  $O\ZstdRegister.obj \
+
+ZSTD_OBJS = \
+  $O\entropy_common.obj \
+  $O\fse_decompress.obj \
+  $O\huf_decompress.obj \
+  $O\zstd_common.obj \
+  $O\zstd_decompress.obj \
+  $O\xxhash.obj \
+  $O\error_private.obj \
+
+ZSTDMT_OBJS = \
+  $O\threading.obj \
+  $O\zstdmt_common.obj \
+  $O\zstdmt_decompress.obj \
+
 !include "../../7zip.mak"

CPP/7zip/Compress/Lz4Decoder.cpp

@@ -0,0 +1,177 @@
+// (C) 2016 Tino Reichardt
+
+#include "StdAfx.h"
+#include "Lz4Decoder.h"
+
+int Lz4Read(void *arg, LZ4MT_Buffer * in)
+{
+  struct Lz4Stream *x = (struct Lz4Stream*)arg;
+  size_t size = in->size;
+
+  HRESULT res = ReadStream(x->inStream, in->buf, &size);
+  if (res != S_OK)
+    return -1;
+
+  in->size = size;
+  *x->processedIn += size;
+
+  return 0;
+}
+
+int Lz4Write(void *arg, LZ4MT_Buffer * out)
+{
+  struct Lz4Stream *x = (struct Lz4Stream*)arg;
+  UInt32 todo = (UInt32)out->size;
+  UInt32 done = 0;
+
+  while (todo != 0)
+  {
+    UInt32 block;
+    HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block);
+    done += block;
+    if (res == k_My_HRESULT_WritingWasCut)
+      break;
+    if (res != S_OK)
+      return -1;
+    if (block == 0)
+      return E_FAIL;
+    todo -= block;
+  }
+
+  *x->processedOut += done;
+  if (x->progress)
+    x->progress->SetRatioInfo(x->processedIn, x->processedOut);
+
+  return 0;
+}
+
+namespace NCompress {
+namespace NLZ4 {
+
+CDecoder::CDecoder():
+  _processedIn(0),
+  _processedOut(0),
+  _inputSize(0),
+  _numThreads(NWindows::NSystem::GetNumberOfProcessors())
+{
+  _props.clear();
+}
+
+CDecoder::~CDecoder()
+{
+}
+
+HRESULT CDecoder::ErrorOut(size_t code)
+{
+  const char *strError = LZ4MT_getErrorString(code);
+  wchar_t wstrError[200+5]; /* no malloc here, /TR */
+
+  mbstowcs(wstrError, strError, 200);
+  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
+  MyFree(wstrError);
+
+  return S_FALSE;
+}
+
+STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
+{
+  DProps *pProps = (DProps *)prop;
+
+  if (size != sizeof(DProps))
+    return E_FAIL;
+
+  memcpy(&_props, pProps, sizeof (DProps));
+
+  return S_OK;
+}
+
+STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
+{
+  const UInt32 kNumThreadsMax = LZ4MT_THREAD_MAX;
+  if (numThreads < 1) numThreads = 1;
+  if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
+  _numThreads = numThreads;
+  return S_OK;
+}
+
+HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
+{
+  _processedOut = 0;
+  return S_OK;
+}
+
+STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
+{
+  _processedIn = 0;
+  RINOK(SetOutStreamSizeResume(outSize));
+  return S_OK;
+}
+
+HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream,
+  ISequentialOutStream * outStream, ICompressProgressInfo * progress)
+{
+  LZ4MT_RdWr_t rdwr;
+  size_t result;
+  HRESULT res = S_OK;
+
+  struct Lz4Stream Rd;
+  Rd.inStream = inStream;
+  Rd.processedIn = &_processedIn;
+
+  struct Lz4Stream Wr;
+  Wr.progress = progress;
+  Wr.outStream = outStream;
+  Wr.processedIn = &_processedIn;
+  Wr.processedOut = &_processedOut;
+
+  /* 1) setup read/write functions */
+  rdwr.fn_read = ::Lz4Read;
+  rdwr.fn_write = ::Lz4Write;
+  rdwr.arg_read = (void *)&Rd;
+  rdwr.arg_write = (void *)&Wr;
+
+  /* 2) create compression context */
+  LZ4MT_DCtx *ctx = LZ4MT_createDCtx(_numThreads, _inputSize);
+  if (!ctx)
+      return S_FALSE;
+
+  /* 3) compress */
+  result = LZ4MT_DecompressDCtx(ctx, &rdwr);
+  if (result == (size_t)-LZ4MT_error_read_fail)
+    res = E_ABORT;
+  else if (LZ4MT_isError(result))
+    return ErrorOut(result);
+
+  /* 4) free resources */
+  LZ4MT_freeDCtx(ctx);
+  return res;
+}
+
+STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
+  const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress)
+{
+  SetOutStreamSize(outSize);
+  return CodeSpec(inStream, outStream, progress);
+}
+
+#ifndef NO_READ_FROM_CODER
+STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
+{
+  _inStream = inStream;
+  return S_OK;
+}
+
+STDMETHODIMP CDecoder::ReleaseInStream()
+{
+  _inStream.Release();
+  return S_OK;
+}
+#endif
+
+HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
+{
+  RINOK(SetOutStreamSizeResume(outSize));
+  return CodeSpec(_inStream, outStream, progress);
+}
+
+}}

CPP/7zip/Compress/Lz4Decoder.h

@@ -0,0 +1,99 @@
+// (C) 2016 Tino Reichardt
+
+/**
+ * you can define LZ4_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7)
+ * /TR 2016-10-01
+ */
+
+#define LZ4_STATIC_LINKING_ONLY
+#include "../../../C/Alloc.h"
+#include "../../../C/Threads.h"
+#include "../../../C/lz4/lz4.h"
+#include "../../../C/zstdmt/lz4mt.h"
+
+#include "../../Windows/System.h"
+#include "../../Common/Common.h"
+#include "../../Common/MyCom.h"
+#include "../ICoder.h"
+#include "../Common/StreamUtils.h"
+#include "../Common/RegisterCodec.h"
+#include "../Common/ProgressMt.h"
+
+struct Lz4Stream {
+  ISequentialInStream *inStream;
+  ISequentialOutStream *outStream;
+  ICompressProgressInfo *progress;
+  UInt64 *processedIn;
+  UInt64 *processedOut;
+  CCriticalSection *cs;
+  int flags;
+};
+
+extern int Lz4Read(void *Stream, LZ4MT_Buffer * in);
+extern int Lz4Write(void *Stream, LZ4MT_Buffer * in);
+
+namespace NCompress {
+namespace NLZ4 {
+
+struct DProps
+{
+  DProps() { clear (); }
+  void clear ()
+  {
+    memset(this, 0, sizeof (*this));
+    _ver_major = LZ4_VERSION_MAJOR;
+    _ver_minor = LZ4_VERSION_MINOR;
+    _level = 1;
+  }
+
+  Byte _ver_major;
+  Byte _ver_minor;
+  Byte _level;
+  Byte _reserved[2];
+};
+
+class CDecoder:public ICompressCoder,
+  public ICompressSetDecoderProperties2,
+  public CMyUnknownImp
+{
+  CMyComPtr < ISequentialInStream > _inStream;
+
+  DProps _props;
+  CCriticalSection cs;
+
+  UInt64 _processedIn;
+  UInt64 _processedOut;
+  UInt32 _inputSize;
+  UInt32 _numThreads;
+
+  HRESULT CDecoder::ErrorOut(size_t code);
+  HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
+  HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
+
+public:
+
+  MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
+  MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2)
+#ifndef NO_READ_FROM_CODER
+  MY_QUERYINTERFACE_ENTRY(ICompressSetInStream)
+#endif
+  MY_QUERYINTERFACE_END
+
+  MY_ADDREF_RELEASE
+  STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size);
+  STDMETHOD (SetOutStreamSize)(const UInt64 *outSize);
+  STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads);
+
+#ifndef NO_READ_FROM_CODER
+  STDMETHOD (SetInStream)(ISequentialInStream *inStream);
+  STDMETHOD (ReleaseInStream)();
+  UInt64 GetInputProcessedSize() const { return _processedIn; }
+#endif
+  HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
+
+  CDecoder();
+  virtual ~CDecoder();
+};
+
+}}

CPP/7zip/Compress/Lz4Encoder.cpp

@@ -0,0 +1,139 @@
+// (C) 2016 Tino Reichardt
+
+#include "StdAfx.h"
+#include "Lz4Encoder.h"
+#include "Lz4Decoder.h"
+
+#ifndef EXTRACT_ONLY
+namespace NCompress {
+namespace NLZ4 {
+
+CEncoder::CEncoder():
+  _processedIn(0),
+  _processedOut(0),
+  _inputSize(0),
+  _ctx(NULL),
+  _numThreads(NWindows::NSystem::GetNumberOfProcessors())
+{
+  _props.clear();
+}
+
+CEncoder::~CEncoder()
+{
+  if (_ctx)
+    LZ4MT_freeCCtx(_ctx);
+}
+
+STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
+{
+  _props.clear();
+
+  for (UInt32 i = 0; i < numProps; i++)
+  {
+    const PROPVARIANT & prop = coderProps[i];
+    PROPID propID = propIDs[i];
+    UInt32 v = (UInt32)prop.ulVal;
+    switch (propID)
+    {
+    case NCoderPropID::kLevel:
+      {
+        if (prop.vt != VT_UI4)
+          return E_INVALIDARG;
+
+        _props._level = static_cast < Byte > (prop.ulVal);
+        Byte lz4_level = static_cast < Byte > (LZ4MT_LEVEL_MAX);
+        if (_props._level > lz4_level)
+          _props._level = lz4_level;
+
+        break;
+      }
+    case NCoderPropID::kNumThreads:
+      {
+        SetNumberOfThreads(v);
+        break;
+      }
+    default:
+      {
+        break;
+      }
+    }
+  }
+
+  return S_OK;
+}
+
+STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream)
+{
+  return WriteStream(outStream, &_props, sizeof (_props));
+}
+
+STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
+  ISequentialOutStream *outStream, const UInt64 * /*inSize*/ ,
+  const UInt64 * /*outSize */, ICompressProgressInfo *progress)
+{
+  LZ4MT_RdWr_t rdwr;
+  size_t result;
+  HRESULT res = S_OK;
+
+  struct Lz4Stream Rd;
+  Rd.inStream = inStream;
+  Rd.outStream = outStream;
+  Rd.processedIn = &_processedIn;
+  Rd.processedOut = &_processedOut;
+
+  struct Lz4Stream Wr;
+  if (_processedIn == 0)
+    Wr.progress = progress;
+  else
+    Wr.progress = 0;
+  Wr.inStream = inStream;
+  Wr.outStream = outStream;
+  Wr.processedIn = &_processedIn;
+  Wr.processedOut = &_processedOut;
+
+  /* 1) setup read/write functions */
+  rdwr.fn_read = ::Lz4Read;
+  rdwr.fn_write = ::Lz4Write;
+  rdwr.arg_read = (void *)&Rd;
+  rdwr.arg_write = (void *)&Wr;
+
+  /* 2) create compression context, if needed */
+  if (!_ctx)
+    _ctx = LZ4MT_createCCtx(_numThreads, _props._level, _inputSize);
+  if (!_ctx)
+    return S_FALSE;
+
+  /* 3) compress */
+  result = LZ4MT_CompressCCtx(_ctx, &rdwr);
+  if (result == (size_t)-LZ4MT_error_read_fail)
+    res = E_ABORT;
+  else if (LZ4MT_isError(result))
+    if (result != LZ4MT_error_read_fail)
+      return ErrorOut(result);
+
+  return res;
+}
+
+STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
+{
+  const UInt32 kNumThreadsMax = LZ4MT_THREAD_MAX;
+  if (numThreads < 1) numThreads = 1;
+  if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
+  _numThreads = numThreads;
+  return S_OK;
+}
+
+HRESULT CEncoder::ErrorOut(size_t code)
+{
+  const char *strError = LZ4MT_getErrorString(code);
+  wchar_t wstrError[200+5]; /* no malloc here, /TR */
+
+  mbstowcs(wstrError, strError, 200);
+  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
+  MyFree(wstrError);
+
+  return S_FALSE;
+}
+
+}}
+#endif

CPP/7zip/Compress/Lz4Encoder.h

@@ -0,0 +1,63 @@
+// (C) 2016 Tino Reichardt
+
+#define LZ4_STATIC_LINKING_ONLY
+#include "../../../C/Alloc.h"
+#include "../../../C/Threads.h"
+#include "../../../C/lz4/lz4.h"
+#include "../../../C/zstdmt/lz4mt.h"
+
+#include "../../Common/Common.h"
+#include "../../Common/MyCom.h"
+#include "../ICoder.h"
+#include "../Common/StreamUtils.h"
+
+#ifndef EXTRACT_ONLY
+namespace NCompress {
+namespace NLZ4 {
+
+struct CProps
+{
+  CProps() { clear (); }
+  void clear ()
+  {
+    memset(this, 0, sizeof (*this));
+    _ver_major = LZ4_VERSION_MAJOR;
+    _ver_minor = LZ4_VERSION_MINOR;
+    _level = 3;
+  }
+
+  Byte _ver_major;
+  Byte _ver_minor;
+  Byte _level;
+  Byte _reserved[2];
+};
+
+class CEncoder:
+  public ICompressCoder,
+  public ICompressSetCoderProperties,
+  public ICompressWriteCoderProperties,
+  public CMyUnknownImp
+{
+  CProps _props;
+
+  UInt64 _processedIn;
+  UInt64 _processedOut;
+  UInt32 _inputSize;
+  UInt32 _numThreads;
+
+  LZ4MT_CCtx *_ctx;
+  HRESULT CEncoder::ErrorOut(size_t code);
+
+public:
+  MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
+  STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
+  STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream);
+  STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads);
+
+  CEncoder();
+  virtual ~CEncoder();
+};
+
+}}
+#endif

CPP/7zip/Compress/Lz4Register.cpp

@@ -0,0 +1,17 @@
+// (C) 2016 Tino Reichardt
+
+#include "StdAfx.h"
+
+#include "../Common/RegisterCodec.h"
+
+#include "Lz4Decoder.h"
+
+#ifndef EXTRACT_ONLY
+#include "Lz4Encoder.h"
+#endif
+
+REGISTER_CODEC_E(
+  LZ4,
+  NCompress::NLZ4::CDecoder(),
+  NCompress::NLZ4::CEncoder(),
+  0x4F71104, "LZ4")

CPP/7zip/Compress/Lz5Decoder.cpp

@@ -0,0 +1,178 @@
+// (C) 2016 Tino Reichardt
+
+#include "StdAfx.h"
+#include "Lz5Decoder.h"
+
+int Lz5Read(void *arg, LZ5MT_Buffer * in)
+{
+  struct Lz5Stream *x = (struct Lz5Stream*)arg;
+  size_t size = in->size;
+
+  HRESULT res = ReadStream(x->inStream, in->buf, &size);
+  if (res != S_OK)
+    return -1;
+
+  in->size = size;
+  *x->processedIn += size;
+
+  return 0;
+}
+
+int Lz5Write(void *arg, LZ5MT_Buffer * out)
+{
+  struct Lz5Stream *x = (struct Lz5Stream*)arg;
+  UInt32 todo = (UInt32)out->size;
+  UInt32 done = 0;
+
+  while (todo != 0)
+  {
+    UInt32 block;
+    HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block);
+    done += block;
+    if (res == k_My_HRESULT_WritingWasCut)
+      break;
+    if (res != S_OK)
+      return -1;
+    if (block == 0)
+      return E_FAIL;
+    todo -= block;
+  }
+
+  *x->processedOut += done;
+  if (x->progress)
+    x->progress->SetRatioInfo(x->processedIn, x->processedOut);
+
+  return 0;
+}
+
+namespace NCompress {
+namespace NLZ5 {
+
+CDecoder::CDecoder():
+  _processedIn(0),
+  _processedOut(0),
+  _inputSize(0),
+  _numThreads(NWindows::NSystem::GetNumberOfProcessors())
+{
+  _props.clear();
+}
+
+CDecoder::~CDecoder()
+{
+}
+
+HRESULT CDecoder::ErrorOut(size_t code)
+{
+  const char *strError = LZ5MT_getErrorString(code);
+  wchar_t wstrError[200+5]; /* no malloc here, /TR */
+
+  mbstowcs(wstrError, strError, 200);
+  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
+  MyFree(wstrError);
+
+  return S_FALSE;
+}
+
+STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
+{
+  DProps *pProps = (DProps *)prop;
+
+  if (size != sizeof(DProps))
+    return E_FAIL;
+
+  memcpy(&_props, pProps, sizeof (DProps));
+
+  return S_OK;
+}
+
+STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
+{
+  const UInt32 kNumThreadsMax = LZ5MT_THREAD_MAX;
+  if (numThreads < 1) numThreads = 1;
+  if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
+  _numThreads = numThreads;
+  return S_OK;
+}
+
+HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
+{
+  _processedOut = 0;
+  return S_OK;
+}
+
+STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
+{
+  _processedIn = 0;
+  RINOK(SetOutStreamSizeResume(outSize));
+  return S_OK;
+}
+
+HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream,
+  ISequentialOutStream * outStream, ICompressProgressInfo * progress)
+{
+  LZ5MT_RdWr_t rdwr;
+  size_t result;
+  HRESULT res = S_OK;
+
+  struct Lz5Stream Rd;
+  Rd.inStream = inStream;
+  Rd.processedIn = &_processedIn;
+
+  struct Lz5Stream Wr;
+  Wr.progress = progress;
+  Wr.outStream = outStream;
+  Wr.processedIn = &_processedIn;
+  Wr.processedOut = &_processedOut;
+
+  /* 1) setup read/write functions */
+  rdwr.fn_read = ::Lz5Read;
+  rdwr.fn_write = ::Lz5Write;
+  rdwr.arg_read = (void *)&Rd;
+  rdwr.arg_write = (void *)&Wr;
+
+  /* 2) create compression context */
+  LZ5MT_DCtx *ctx = LZ5MT_createDCtx(_numThreads, _inputSize);
+  if (!ctx)
+      return S_FALSE;
+
+  /* 3) compress */
+  result = LZ5MT_DecompressDCtx(ctx, &rdwr);
+  if (result == (size_t)-LZ5MT_error_read_fail)
+    res = E_ABORT;
+  else if (LZ5MT_isError(result))
+    if (result != LZ5MT_error_read_fail)
+      return ErrorOut(result);
+
+  /* 4) free resources */
+  LZ5MT_freeDCtx(ctx);
+  return res;
+}
+
+STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
+  const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress)
+{
+  SetOutStreamSize(outSize);
+  return CodeSpec(inStream, outStream, progress);
+}
+
+#ifndef NO_READ_FROM_CODER
+STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
+{
+  _inStream = inStream;
+  return S_OK;
+}
+
+STDMETHODIMP CDecoder::ReleaseInStream()
+{
+  _inStream.Release();
+  return S_OK;
+}
+#endif
+
+HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
+{
+  RINOK(SetOutStreamSizeResume(outSize));
+  return CodeSpec(_inStream, outStream, progress);
+}
+
+}}

CPP/7zip/Compress/Lz5Decoder.h

@@ -0,0 +1,99 @@
+// (C) 2016 Tino Reichardt
+
+/**
+ * you can define LZ5_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7)
+ * /TR 2016-10-01
+ */
+
+#define LZ5_STATIC_LINKING_ONLY
+#include "../../../C/Alloc.h"
+#include "../../../C/Threads.h"
+#include "../../../C/lz5/lz5.h"
+#include "../../../C/zstdmt/lz5mt.h"
+
+#include "../../Windows/System.h"
+#include "../../Common/Common.h"
+#include "../../Common/MyCom.h"
+#include "../ICoder.h"
+#include "../Common/StreamUtils.h"
+#include "../Common/RegisterCodec.h"
+#include "../Common/ProgressMt.h"
+
+struct Lz5Stream {
+  ISequentialInStream *inStream;
+  ISequentialOutStream *outStream;
+  ICompressProgressInfo *progress;
+  UInt64 *processedIn;
+  UInt64 *processedOut;
+  CCriticalSection *cs;
+  int flags;
+};
+
+extern int Lz5Read(void *Stream, LZ5MT_Buffer * in);
+extern int Lz5Write(void *Stream, LZ5MT_Buffer * in);
+
+namespace NCompress {
+namespace NLZ5 {
+
+struct DProps
+{
+  DProps() { clear (); }
+  void clear ()
+  {
+    memset(this, 0, sizeof (*this));
+    _ver_major = LZ5_VERSION_MAJOR;
+    _ver_minor = LZ5_VERSION_MINOR;
+    _level = 1;
+  }
+
+  Byte _ver_major;
+  Byte _ver_minor;
+  Byte _level;
+  Byte _reserved[2];
+};
+
+class CDecoder:public ICompressCoder,
+  public ICompressSetDecoderProperties2,
+  public CMyUnknownImp
+{
+  CMyComPtr < ISequentialInStream > _inStream;
+
+  DProps _props;
+  CCriticalSection cs;
+
+  UInt64 _processedIn;
+  UInt64 _processedOut;
+  UInt32 _inputSize;
+  UInt32 _numThreads;
+
+  HRESULT CDecoder::ErrorOut(size_t code);
+  HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
+  HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
+
+public:
+
+  MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
+  MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2)
+#ifndef NO_READ_FROM_CODER
+  MY_QUERYINTERFACE_ENTRY(ICompressSetInStream)
+#endif
+  MY_QUERYINTERFACE_END
+
+  MY_ADDREF_RELEASE
+  STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size);
+  STDMETHOD (SetOutStreamSize)(const UInt64 *outSize);
+  STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads);
+
+#ifndef NO_READ_FROM_CODER
+  STDMETHOD (SetInStream)(ISequentialInStream *inStream);
+  STDMETHOD (ReleaseInStream)();
+  UInt64 GetInputProcessedSize() const { return _processedIn; }
+#endif
+  HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
+
+  CDecoder();
+  virtual ~CDecoder();
+};
+
+}}

CPP/7zip/Compress/Lz5Encoder.cpp

@@ -0,0 +1,138 @@
+// (C) 2016 Tino Reichardt
+
+#include "StdAfx.h"
+#include "Lz5Encoder.h"
+#include "Lz5Decoder.h"
+
+#ifndef EXTRACT_ONLY
+namespace NCompress {
+namespace NLZ5 {
+
+CEncoder::CEncoder():
+  _processedIn(0),
+  _processedOut(0),
+  _inputSize(0),
+  _ctx(NULL),
+  _numThreads(NWindows::NSystem::GetNumberOfProcessors())
+{
+  _props.clear();
+}
+
+CEncoder::~CEncoder()
+{
+  if (_ctx)
+    LZ5MT_freeCCtx(_ctx);
+}
+
+STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
+{
+  _props.clear();
+
+  for (UInt32 i = 0; i < numProps; i++)
+  {
+    const PROPVARIANT & prop = coderProps[i];
+    PROPID propID = propIDs[i];
+    UInt32 v = (UInt32)prop.ulVal;
+    switch (propID)
+    {
+    case NCoderPropID::kLevel:
+      {
+        if (prop.vt != VT_UI4)
+          return E_INVALIDARG;
+
+        _props._level = static_cast < Byte > (prop.ulVal);
+        Byte lz5_level = static_cast < Byte > (LZ5MT_LEVEL_MAX);
+        if (_props._level > lz5_level)
+          _props._level = lz5_level;
+
+        break;
+      }
+    case NCoderPropID::kNumThreads:
+      {
+        SetNumberOfThreads(v);
+        break;
+      }
+    default:
+      {
+        break;
+      }
+    }
+  }
+
+  return S_OK;
+}
+
+STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream)
+{
+  return WriteStream(outStream, &_props, sizeof (_props));
+}
+
+STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
+  ISequentialOutStream *outStream, const UInt64 * /*inSize*/ ,
+  const UInt64 * /*outSize */, ICompressProgressInfo *progress)
+{
+  LZ5MT_RdWr_t rdwr;
+  size_t result;
+  HRESULT res = S_OK;
+
+  struct Lz5Stream Rd;
+  Rd.inStream = inStream;
+  Rd.outStream = outStream;
+  Rd.processedIn = &_processedIn;
+  Rd.processedOut = &_processedOut;
+
+  struct Lz5Stream Wr;
+  if (_processedIn == 0)
+    Wr.progress = progress;
+  else
+    Wr.progress = 0;
+  Wr.inStream = inStream;
+  Wr.outStream = outStream;
+  Wr.processedIn = &_processedIn;
+  Wr.processedOut = &_processedOut;
+
+  /* 1) setup read/write functions */
+  rdwr.fn_read = ::Lz5Read;
+  rdwr.fn_write = ::Lz5Write;
+  rdwr.arg_read = (void *)&Rd;
+  rdwr.arg_write = (void *)&Wr;
+
+  /* 2) create compression context, if needed */
+  if (!_ctx)
+    _ctx = LZ5MT_createCCtx(_numThreads, _props._level, _inputSize);
+  if (!_ctx)
+    return S_FALSE;
+
+  /* 3) compress */
+  result = LZ5MT_CompressCCtx(_ctx, &rdwr);
+  if (result == (size_t)-LZ5MT_error_read_fail)
+    res = E_ABORT;
+  if (LZ5MT_isError(result))
+    return ErrorOut(result);
+
+  return res;
+}
+
+STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
+{
+  const UInt32 kNumThreadsMax = LZ5MT_THREAD_MAX;
+  if (numThreads < 1) numThreads = 1;
+  if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
+  _numThreads = numThreads;
+  return S_OK;
+}
+
+HRESULT CEncoder::ErrorOut(size_t code)
+{
+  const char *strError = LZ5MT_getErrorString(code);
+  wchar_t wstrError[200+5]; /* no malloc here, /TR */
+
+  mbstowcs(wstrError, strError, 200);
+  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
+  MyFree(wstrError);
+
+  return S_FALSE;
+}
+
+}}
+#endif

CPP/7zip/Compress/Lz5Encoder.h

@@ -0,0 +1,63 @@
+// (C) 2016 Tino Reichardt
+
+#define LZ5_STATIC_LINKING_ONLY
+#include "../../../C/Alloc.h"
+#include "../../../C/Threads.h"
+#include "../../../C/lz5/lz5.h"
+#include "../../../C/zstdmt/lz5mt.h"
+
+#include "../../Common/Common.h"
+#include "../../Common/MyCom.h"
+#include "../ICoder.h"
+#include "../Common/StreamUtils.h"
+
+#ifndef EXTRACT_ONLY
+namespace NCompress {
+namespace NLZ5 {
+
+struct CProps
+{
+  CProps() { clear (); }
+  void clear ()
+  {
+    memset(this, 0, sizeof (*this));
+    _ver_major = LZ5_VERSION_MAJOR;
+    _ver_minor = LZ5_VERSION_MINOR;
+    _level = 3;
+  }
+
+  Byte _ver_major;
+  Byte _ver_minor;
+  Byte _level;
+  Byte _reserved[2];
+};
+
+class CEncoder:
+  public ICompressCoder,
+  public ICompressSetCoderProperties,
+  public ICompressWriteCoderProperties,
+  public CMyUnknownImp
+{
+  CProps _props;
+
+  UInt64 _processedIn;
+  UInt64 _processedOut;
+  UInt32 _inputSize;
+  UInt32 _numThreads;
+
+  LZ5MT_CCtx *_ctx;
+  HRESULT CEncoder::ErrorOut(size_t code);
+
+public:
+  MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
+  STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
+  STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream);
+  STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads);
+
+  CEncoder();
+  virtual ~CEncoder();
+};
+
+}}
+#endif

CPP/7zip/Compress/Lz5Register.cpp

@@ -0,0 +1,17 @@
+// (C) 2016 Tino Reichardt
+
+#include "StdAfx.h"
+
+#include "../Common/RegisterCodec.h"
+
+#include "Lz5Decoder.h"
+
+#ifndef EXTRACT_ONLY
+#include "Lz5Encoder.h"
+#endif
+
+REGISTER_CODEC_E(
+  LZ5,
+  NCompress::NLZ5::CDecoder(),
+  NCompress::NLZ5::CEncoder(),
+  0x4F71105, "LZ5")

CPP/7zip/Compress/ZstdDecoder.cpp

@@ -1,33 +1,76 @@
-// ZstdDecoder.cpp
 // (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 #include "ZstdDecoder.h"
 
+int ZstdRead(void *arg, ZSTDMT_Buffer * in)
+{
+  struct ZstdStream *x = (struct ZstdStream*)arg;
+  size_t size = in->size;
+
+  HRESULT res = ReadStream(x->inStream, in->buf, &size);
+  if (res != S_OK)
+    return -1;
+
+  in->size = size;
+  *x->processedIn += size;
+
+  return 0;
+}
+
+int ZstdWrite(void *arg, ZSTDMT_Buffer * out)
+{
+  struct ZstdStream *x = (struct ZstdStream*)arg;
+  UInt32 todo = (UInt32)out->size;
+  UInt32 done = 0;
+
+  while (todo != 0)
+  {
+    UInt32 block;
+    HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block);
+    done += block;
+    if (res == k_My_HRESULT_WritingWasCut)
+      break;
+    if (res != S_OK)
+      return -1;
+    if (block == 0)
+      return E_FAIL;
+    todo -= block;
+  }
+
+  *x->processedOut += done;
+  if (x->progress)
+    x->progress->SetRatioInfo(x->processedIn, x->processedOut);
+
+  return 0;
+}
+
 namespace NCompress {
 namespace NZSTD {
 
 CDecoder::CDecoder():
-  _dstream(NULL),
-  _buffIn(NULL),
-  _buffOut(NULL),
-  _buffInSizeAllocated(0),
-  _buffOutSizeAllocated(0),
-  _buffInSize(ZSTD_DStreamInSize()),
-  _buffOutSize(ZSTD_DStreamOutSize()*4),
   _processedIn(0),
-  _processedOut(0)
+  _processedOut(0),
+  _inputSize(0),
+  _numThreads(NWindows::NSystem::GetNumberOfProcessors())
 {
   _props.clear();
 }
 
 CDecoder::~CDecoder()
 {
-  if (_dstream)
+}
-    ZSTD_freeDStream(_dstream);
 
-  MyFree(_buffIn);
+HRESULT CDecoder::ErrorOut(size_t code)
-  MyFree(_buffOut);
+{
+  const char *strError = ZSTDMT_getErrorString(code);
+  wchar_t wstrError[200+5]; /* no malloc here, /TR */
+
+  mbstowcs(wstrError, strError, 200);
+  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
+  MyFree(wstrError);
+
+  return S_FALSE;
 }
 
 STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
@@ -37,97 +80,23 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
   if (size != sizeof(DProps))
     return E_FAIL;
 
-#ifdef ZSTD_LEGACY_SUPPORT
-  /* version 0.x and 1.x are okay */
-  if (pProps->_ver_major > 1)
-    return E_FAIL;
-
-  /* 0.5, 0.6, 0.7, 0.8 are supported! */
-  if (pProps->_ver_major == 0) {
-    switch (pProps->_ver_minor) {
-    case 5:
-      break;
-    case 6:
-      break;
-    case 7:
-      break;
-    case 8:
-      break;
-    default:
-      return E_FAIL;
-  }}
-#else
-  /* only exact version is okay */
-  if (pProps->_ver_major != ZSTD_VERSION_MAJOR)
-    return E_FAIL;
-  if (pProps->_ver_minor != ZSTD_VERSION_MINOR)
-    return E_FAIL;
-#endif
-
   memcpy(&_props, pProps, sizeof (DProps));
 
   return S_OK;
 }
 
-HRESULT CDecoder::ErrorOut(size_t code)
+STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads)
 {
-  const char *strError = ZSTD_getErrorName(code);
+  const UInt32 kNumThreadsMax = ZSTDMT_THREAD_MAX;
-  size_t strErrorLen = strlen(strError) + 1;
+  if (numThreads < 1) numThreads = 1;
-  wchar_t *wstrError = (wchar_t *)MyAlloc(sizeof(wchar_t) * strErrorLen);
+  if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
-
+  _numThreads = numThreads;
-  if (!wstrError)
-    return E_FAIL;
-
-  mbstowcs(wstrError, strError, strErrorLen - 1);
-  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
-  MyFree(wstrError);
-
-  return E_FAIL;
-}
-
-HRESULT CDecoder::CreateDecompressor()
-{
-  size_t result;
-
-  if (!_dstream) {
-    _dstream = ZSTD_createDStream();
-    if (!_dstream)
-      return E_FAIL;
-  }
-
-  result = ZSTD_initDStream(_dstream);
-  if (ZSTD_isError(result))
-    return ErrorOut(result);
-
-  /* allocate buffers */
-  if (_buffInSizeAllocated != _buffInSize)
-  {
-    MyFree(_buffIn);
-    _buffIn = MyAlloc(_buffInSize);
-
-    if (!_buffIn)
-      return E_OUTOFMEMORY;
-    _buffInSizeAllocated = _buffInSize;
-  }
-
-  if (_buffOutSizeAllocated != _buffOutSize)
-  {
-    MyFree(_buffOut);
-    _buffOut = MyAlloc(_buffOutSize);
-
-    if (!_buffOut)
-      return E_OUTOFMEMORY;
-    _buffOutSizeAllocated = _buffOutSize;
-  }
-
   return S_OK;
 }
 
 HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
 {
   _processedOut = 0;
-  RINOK(CreateDecompressor());
-
   return S_OK;
 }
 
@@ -135,73 +104,48 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
 {
   _processedIn = 0;
   RINOK(SetOutStreamSizeResume(outSize));
-
   return S_OK;
 }
 
-STDMETHODIMP CDecoder::SetInBufSize(UInt32, UInt32 size)
+HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream,
+  ISequentialOutStream * outStream, ICompressProgressInfo * progress)
 {
-  _buffInSize = size;
+  ZSTDMT_RdWr_t rdwr;
-  return S_OK;
-}
-
-STDMETHODIMP CDecoder::SetOutBufSize(UInt32, UInt32 size)
-{
-  _buffOutSize = size;
-  return S_OK;
-}
-
-HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress)
-{
-  RINOK(CreateDecompressor());
-
   size_t result;
-  UInt32 const toRead = static_cast < const UInt32 > (_buffInSize);
+  HRESULT res = S_OK;
-  for(;;) {
-    UInt32 read;
 
-    /* read input */
+  struct ZstdStream Rd;
-    RINOK(inStream->Read(_buffIn, toRead, &read));
+  Rd.inStream = inStream;
-    size_t InSize = static_cast < size_t > (read);
+  Rd.processedIn = &_processedIn;
-    _processedIn += InSize;
 
-    if (InSize == 0)
+  struct ZstdStream Wr;
-      return S_OK;
+  Wr.progress = progress;
+  Wr.outStream = outStream;
+  Wr.processedIn = &_processedIn;
+  Wr.processedOut = &_processedOut;
 
-    /* decompress input */
+  /* 1) setup read/write functions */
-    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
+  rdwr.fn_read = ::ZstdRead;
-    for (;;) {
+  rdwr.fn_write = ::ZstdWrite;
-      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+  rdwr.arg_read = (void *)&Rd;
-      result = ZSTD_decompressStream(_dstream, &output , &input);
+  rdwr.arg_write = (void *)&Wr;
-      #if 0
+
-      printf("%s in=%d out=%d result=%d in.pos=%d in.size=%d\n", __FUNCTION__,
+  /* 2) create decompression context */
-        InSize, output.pos, result, input.pos, input.size);
+  ZSTDMT_DCtx *ctx = ZSTDMT_createDCtx(_numThreads, _inputSize);
-      fflush(stdout);
+  if (!ctx)
-      #endif
+      return S_FALSE;
-      if (ZSTD_isError(result))
+
+  /* 3) decompress */
+  result = ZSTDMT_decompressDCtx(ctx, &rdwr);
+  //printf("decompress = %d / %d\n", result, ZSTDMT_error_read_fail);
+  if (result == (size_t)-ZSTDMT_error_read_fail)
+    res = E_ABORT;
+  else if (ZSTDMT_isError(result))
     return ErrorOut(result);
 
-      /* write decompressed stream and update progress */
+  /* 4) free resources */
-      RINOK(WriteStream(outStream, _buffOut, output.pos));
+  ZSTDMT_freeDCtx(ctx);
-      _processedOut += output.pos;
+  return res;
-      RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
-
-      /* one more round */
-      if ((input.pos == input.size) && (result == 1) && output.pos)
-        continue;
-
-      /* finished */
-      if (input.pos == input.size)
-        break;
-
-      /* end of frame */
-      if (result == 0) {
-        result = ZSTD_initDStream(_dstream);
-        if (ZSTD_isError(result))
-          return ErrorOut(result);
-      }
-    }
-  }
 }
 
 STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
@@ -221,69 +165,14 @@ STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
 STDMETHODIMP CDecoder::ReleaseInStream()
 {
   _inStream.Release();
-
   return S_OK;
 }
-
+#endif
-STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize)
-{
-  if (processedSize)
-    *processedSize = 0;
-
-  size_t result;
-
-  if (!_dstream)
-    if (CreateDecompressor() != S_OK)
-      return E_FAIL;
-
-  UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
-  Byte *dataout = static_cast < Byte* > (data);
-  for(;;) {
-    /* read input */
-    RINOK(_inStream->Read(_buffIn, toRead, &read));
-    size_t InSize = static_cast < size_t > (read);
-    _processedIn += InSize;
-
-    if (InSize == 0) {
-      return S_OK;
-    }
-
-    /* decompress input */
-    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
-    for (;;) {
-      ZSTD_outBuffer output = { dataout, _buffOutSize, 0 };
-      result = ZSTD_decompressStream(_dstream, &output , &input);
-      if (ZSTD_isError(result))
-        return ErrorOut(result);
-
-      if (processedSize)
-        *processedSize += static_cast < UInt32 > (output.pos);
-
-      dataout += output.pos;
-
-      /* one more round */
-      if ((input.pos == input.size) && (result == 1) && output.pos)
-        continue;
-
-      /* finished */
-      if (input.pos == input.size)
-        break;
-
-      /* end of frame */
-      if (result == 0) {
-        result = ZSTD_initDStream(_dstream);
-        if (ZSTD_isError(result))
-          return ErrorOut(result);
-      }
-    }
-  }
-}
 
 HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
 {
   RINOK(SetOutStreamSizeResume(outSize));
   return CodeSpec(_inStream, outStream, progress);
 }
-#endif
 
 }}

CPP/7zip/Compress/ZstdDecoder.h

@@ -1,22 +1,36 @@
-// ZstdDecoder.h
 // (C) 2016 Tino Reichardt
 
 /**
- * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible
+ * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible (0.1 .. 0.7)
- * with these versions: 0.5, 0.6, 0.7, 0.8 (0.8 == 1.0)
+ * /TR 2016-10-01
- *
- * /TR 2016-09-04
  */
 
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../../../C/Alloc.h"
-#include "../../../C/ZStd/zstd.h"
+#include "../../../C/Threads.h"
+#include "../../../C/zstd/zstd.h"
+#include "../../../C/zstdmt/zstdmt.h"
 
+#include "../../Windows/System.h"
 #include "../../Common/Common.h"
 #include "../../Common/MyCom.h"
 #include "../ICoder.h"
 #include "../Common/StreamUtils.h"
 #include "../Common/RegisterCodec.h"
+#include "../Common/ProgressMt.h"
+
+struct ZstdStream {
+  ISequentialInStream *inStream;
+  ISequentialOutStream *outStream;
+  ICompressProgressInfo *progress;
+  UInt64 *processedIn;
+  UInt64 *processedOut;
+  CCriticalSection *cs;
+  int flags;
+};
+
+extern int ZstdRead(void *Stream, ZSTDMT_Buffer * in);
+extern int ZstdWrite(void *Stream, ZSTDMT_Buffer * in);
 
 namespace NCompress {
 namespace NZSTD {
@@ -39,60 +53,44 @@ struct DProps
 };
 
 class CDecoder:public ICompressCoder,
-  public ICompressSetDecoderProperties2, public ICompressSetBufSize,
+  public ICompressSetDecoderProperties2,
-#ifndef NO_READ_FROM_CODER
-  public ICompressSetInStream,
-  public ICompressSetOutStreamSize, public ISequentialInStream,
-#endif
   public CMyUnknownImp
 {
   CMyComPtr < ISequentialInStream > _inStream;
 
   DProps _props;
-
+  CCriticalSection cs;
-  ZSTD_DStream *_dstream;
-  void *_buffIn;
-  void *_buffOut;
-
-  size_t _buffInSize;
-  size_t _buffOutSize;
-  size_t _buffInSizeAllocated;
-  size_t _buffOutSizeAllocated;
 
   UInt64 _processedIn;
   UInt64 _processedOut;
+  UInt32 _inputSize;
+  UInt32 _numThreads;
 
-  HRESULT CDecoder::CreateDecompressor();
   HRESULT CDecoder::ErrorOut(size_t code);
   HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
   HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
 
 public:
 
-  MY_QUERYINTERFACE_BEGIN2 (ICompressCoder)
+  MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
-  MY_QUERYINTERFACE_ENTRY (ICompressSetDecoderProperties2)
+  MY_QUERYINTERFACE_ENTRY(ICompressSetDecoderProperties2)
-  MY_QUERYINTERFACE_ENTRY (ICompressSetBufSize)
 #ifndef NO_READ_FROM_CODER
-  MY_QUERYINTERFACE_ENTRY (ICompressSetInStream)
+  MY_QUERYINTERFACE_ENTRY(ICompressSetInStream)
-  MY_QUERYINTERFACE_ENTRY (ICompressSetOutStreamSize)
-  MY_QUERYINTERFACE_ENTRY (ISequentialInStream)
 #endif
   MY_QUERYINTERFACE_END
 
   MY_ADDREF_RELEASE
   STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
-  STDMETHOD (SetDecoderProperties2) (const Byte *data, UInt32 size);
+  STDMETHOD (SetDecoderProperties2)(const Byte *data, UInt32 size);
-  STDMETHOD (SetOutStreamSize) (const UInt64 *outSize);
+  STDMETHOD (SetOutStreamSize)(const UInt64 *outSize);
-  STDMETHOD (SetInBufSize) (UInt32 streamIndex, UInt32 size);
+  STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads);
-  STDMETHOD (SetOutBufSize) (UInt32 streamIndex, UInt32 size);
 
 #ifndef NO_READ_FROM_CODER
-  STDMETHOD (SetInStream) (ISequentialInStream *inStream);
+  STDMETHOD (SetInStream)(ISequentialInStream *inStream);
-  STDMETHOD (ReleaseInStream) ();
+  STDMETHOD (ReleaseInStream)();
-  STDMETHOD (Read) (void *data, UInt32 size, UInt32 *processedSize);
+  UInt64 GetInputProcessedSize() const { return _processedIn; }
-  HRESULT CodeResume (ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
-  UInt64 GetInputProcessedSize () const { return _processedIn; }
 #endif
+  HRESULT CodeResume(ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
 
   CDecoder();
   virtual ~CDecoder();

CPP/7zip/Compress/ZstdEncoder.cpp

@@ -1,32 +1,27 @@
-// ZstdEncoder.cpp
 // (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 #include "ZstdEncoder.h"
+#include "ZstdDecoder.h"
 
 #ifndef EXTRACT_ONLY
 namespace NCompress {
 namespace NZSTD {
 
 CEncoder::CEncoder():
-  _cstream(NULL),
-  _buffIn(NULL),
-  _buffOut(NULL),
-  _buffInSize(0),
-  _buffOutSize(0),
   _processedIn(0),
-  _processedOut(0)
+  _processedOut(0),
+  _inputSize(0),
+  _ctx(NULL),
+  _numThreads(NWindows::NSystem::GetNumberOfProcessors())
 {
   _props.clear();
 }
 
 CEncoder::~CEncoder()
 {
-  if (_cstream)
+  if (_ctx)
-    ZSTD_freeCStream(_cstream);
+    ZSTDMT_freeCCtx(_ctx);
-
-  MyFree(_buffIn);
-  MyFree(_buffOut);
 }
 
 STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
@@ -37,6 +32,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
   {
     const PROPVARIANT & prop = coderProps[i];
     PROPID propID = propIDs[i];
+    UInt32 v = (UInt32)prop.ulVal;
     switch (propID)
     {
     case NCoderPropID::kLevel:
@@ -52,6 +48,11 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
 
         break;
       }
+    case NCoderPropID::kNumThreads:
+      {
+        SetNumberOfThreads(v);
+        break;
+      }
     default:
       {
         break;
@@ -59,9 +60,6 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI
     }
   }
 
-  _processedIn = 0;
-  _processedOut = 0;
-
   return S_OK;
 }
 
@@ -71,75 +69,70 @@ STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream)
 }
 
 STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
-  ISequentialOutStream *outStream, const UInt64 * /* inSize */ ,
+  ISequentialOutStream *outStream, const UInt64 * /*inSize*/ ,
-  const UInt64 * /* outSize */ , ICompressProgressInfo *progress)
+  const UInt64 * /*outSize */, ICompressProgressInfo *progress)
 {
+  ZSTDMT_RdWr_t rdwr;
   size_t result;
+  HRESULT res = S_OK;
 
-  /* init only once in beginning */
+  struct ZstdStream Rd;
-  if (!_cstream) {
+  Rd.inStream = inStream;
+  Rd.outStream = outStream;
+  Rd.processedIn = &_processedIn;
+  Rd.processedOut = &_processedOut;
 
-    /* allocate stream */
+  struct ZstdStream Wr;
-    _cstream = ZSTD_createCStream();
+  if (_processedIn == 0)
-    if (!_cstream)
+    Wr.progress = progress;
-      return E_OUTOFMEMORY;
+  else
+    Wr.progress = 0;
+  Wr.inStream = inStream;
+  Wr.outStream = outStream;
+  Wr.processedIn = &_processedIn;
+  Wr.processedOut = &_processedOut;
 
-    /* allocate buffers */
+  /* 1) setup read/write functions */
-    _buffInSize = ZSTD_CStreamInSize();
+  rdwr.fn_read = ::ZstdRead;
-    _buffIn = MyAlloc(_buffInSize);
+  rdwr.fn_write = ::ZstdWrite;
-    if (!_buffIn)
+  rdwr.arg_read = (void *)&Rd;
-      return E_OUTOFMEMORY;
+  rdwr.arg_write = (void *)&Wr;
 
-    _buffOutSize = ZSTD_CStreamOutSize();
+  /* 2) create compression context, if needed */
-    _buffOut = MyAlloc(_buffOutSize);
+  if (!_ctx)
-    if (!_buffOut)
+    _ctx = ZSTDMT_createCCtx(_numThreads, _props._level, _inputSize);
-      return E_OUTOFMEMORY;
+  if (!_ctx)
-  }
-
-  /* init or re-init stream */
-  result = ZSTD_initCStream(_cstream, _props._level);
-  if (ZSTD_isError(result))
     return S_FALSE;
 
-  UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
+  /* 3) compress */
-  for(;;) {
+  result = ZSTDMT_compressCCtx(_ctx, &rdwr);
+  if (result == (size_t)-ZSTDMT_error_read_fail)
+    res = E_ABORT;
+  else if (ZSTDMT_isError(result))
+    return ErrorOut(result);
 
-    /* read input */
+  return res;
-    RINOK(inStream->Read(_buffIn, toRead, &read));
+}
-    size_t InSize = static_cast < size_t > (read);
-    _processedIn += InSize;
-
-    if (InSize == 0) {
-
-      /* @eof */
-      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
-      result = ZSTD_endStream(_cstream, &output);
-      if (ZSTD_isError(result))
-        return S_FALSE;
-
-      if (output.pos) {
-        /* write last compressed bytes and update progress */
-        RINOK(WriteStream(outStream, _buffOut, output.pos));
-        _processedOut += output.pos;
-        RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
-      }
 
+STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads)
+{
+  const UInt32 kNumThreadsMax = ZSTDMT_THREAD_MAX;
+  if (numThreads < 1) numThreads = 1;
+  if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax;
+  _numThreads = numThreads;
   return S_OK;
-    }
+}
+
+HRESULT CEncoder::ErrorOut(size_t code)
+{
+  const char *strError = ZSTDMT_getErrorString(code);
+  wchar_t wstrError[200+5]; /* no malloc here, /TR */
+
+  mbstowcs(wstrError, strError, 200);
+  MessageBoxW(0, wstrError, L"7-Zip ZStandard", MB_ICONERROR | MB_OK);
+  MyFree(wstrError);
 
-    /* compress input */
-    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
-    while (input.pos < input.size) {
-      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
-      result = ZSTD_compressStream(_cstream, &output , &input);
-      if (ZSTD_isError(result))
   return S_FALSE;
-      /* write compressed stream and update progress */
-      RINOK(WriteStream(outStream, _buffOut, output.pos));
-      _processedOut += output.pos;
-      RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
-    }
-  }
 }
 
 }}

CPP/7zip/Compress/ZstdEncoder.h

@@ -1,9 +1,10 @@
-// ZstdEncoder.h
 // (C) 2016 Tino Reichardt
 
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../../../C/Alloc.h"
-#include "../../../C/ZStd/zstd.h"
+#include "../../../C/Threads.h"
+#include "../../../C/zstd/zstd.h"
+#include "../../../C/zstdmt/zstdmt.h"
 
 #include "../../Common/Common.h"
 #include "../../Common/MyCom.h"
@@ -39,21 +40,20 @@ class CEncoder:
 {
   CProps _props;
 
-  ZSTD_CStream *_cstream;
-  void *_buffIn;
-  void *_buffOut;
-  size_t _buffInSize;
-  size_t _buffOutSize;
   UInt64 _processedIn;
   UInt64 _processedOut;
+  UInt32 _inputSize;
+  UInt32 _numThreads;
 
-  HRESULT CreateCompressor();
+  ZSTDMT_CCtx *_ctx;
+  HRESULT CEncoder::ErrorOut(size_t code);
 
 public:
   MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
-  STDMETHOD (Code) (ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
-  STDMETHOD (SetCoderProperties) (const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
+  STDMETHOD (SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
-  STDMETHOD (WriteCoderProperties) (ISequentialOutStream *outStream);
+  STDMETHOD (WriteCoderProperties)(ISequentialOutStream *outStream);
+  STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads);
 
   CEncoder();
   virtual ~CEncoder();

CPP/7zip/Compress/ZstdRegister.cpp

@@ -1,4 +1,3 @@
-// ZstdRegister.cpp
 // (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"

CPP/7zip/Guid.txt

@@ -161,8 +161,11 @@ Handler GUIDs:
   0A lzma
   0B lzma86
   0C xz
-  0D ppmd
+  0E zstd
+  0F lz4
+  10 lz5
 
+  C6 Lzip
   C7 Ext
   C8 VMDK
   C9 VDI

CPP/7zip/UI/Common/OpenArchive.cpp

@@ -1066,6 +1066,7 @@ static const char * const k_Formats_with_simple_signuature[] =
   , "rar"
   , "bzip2"
   , "gzip"
+  , "lzip"
   , "cab"
   , "wim"
   , "rpm"

CPP/7zip/UI/Explorer/ContextMenu.cpp

@@ -283,8 +283,12 @@ static const char * const kArcExts[] =
     "7z"
   , "bz2"
   , "gz"
+  , "lz"
+  , "lz4"
+  , "lz5"
   , "rar"
   , "zip"
+  , "zst"
 };
 
 static bool IsItArcExt(const UString &ext)

CPP/7zip/UI/GUI/CompressDialog.cpp

@@ -87,13 +87,9 @@ static const UInt32 g_Levels[] =
 {
   IDS_METHOD_STORE,
   IDS_METHOD_FASTEST,
-  0,
   IDS_METHOD_FAST,
-  0,
   IDS_METHOD_NORMAL,
-  0,
   IDS_METHOD_MAXIMUM,
-  0,
   IDS_METHOD_ULTRA
 };
 
@@ -101,6 +97,8 @@ enum EMethodID
 {
   kCopy,
   kZSTD,
+  kLZ4,
+  kLZ5,
   kLZMA,
   kLZMA2,
   kPPMd,
@@ -114,6 +112,8 @@ static const LPCWSTR kMethodsNames[] =
 {
   L"Copy",
   L"ZSTD",
+  L"LZ4",
+  L"LZ5",
   L"LZMA",
   L"LZMA2",
   L"PPMd",
@@ -126,6 +126,8 @@ static const LPCWSTR kMethodsNames[] =
 static const EMethodID g_7zMethods[] =
 {
   kZSTD,
+  kLZ4,
+  kLZ5,
   kLZMA2,
   kLZMA,
   kPPMd,
@@ -167,6 +169,21 @@ static const EMethodID g_XzMethods[] =
   kLZMA2
 };
 
+static const EMethodID g_ZstdMethods[] =
+{
+  kZSTD
+};
+
+static const EMethodID g_Lz4Methods[] =
+{
+  kLZ4
+};
+
+static const EMethodID g_Lz5Methods[] =
+{
+  kLZ5
+};
+
 static const EMethodID g_SwfcMethods[] =
 {
   kDeflate
@@ -227,6 +244,24 @@ static const CFormatInfo g_Formats[] =
     METHODS_PAIR(g_XzMethods),
     false, false, true, false, false, false
   },
+  {
+    L"zstd",
+    (1 << 0) | (1 << 1) | (1 << 5) | (1 << 11) | (1 << 17) | (1 << 22),
+    METHODS_PAIR(g_ZstdMethods),
+    false, false, true, false, false, false
+  },
+  {
+    L"lz4",
+    (1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 16),
+    METHODS_PAIR(g_Lz4Methods),
+    false, false, true, false, false, false
+    },
+  {
+    L"lz5",
+    (1 << 0) | (1 << 1) | (1 << 3) | (1 << 7) | (1 << 11) | (1 << 16),
+    METHODS_PAIR(g_Lz5Methods),
+    false, false, true, false, false, false
+  },
   {
     L"Swfc",
     (1 << 1) | (1 << 3) | (1 << 5) | (1 << 7) | (1 << 9),
@@ -867,10 +902,6 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam)
       }
       case IDC_COMPRESS_LEVEL:
       {
-        const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
-        int index = FindRegistryFormatAlways(ai.Name);
-        NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
-        fo.ResetForLevelChange();
         SetMethod(GetMethodID());
         SetSolidBlockSize();
         SetNumThreads();
@@ -880,13 +911,7 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam)
       }
       case IDC_COMPRESS_METHOD:
       {
-        // MessageBoxW(*this, L"IDC_COMPRESS_METHOD!", L"7-Zip", MB_ICONERROR);
+        SetLevel();
-        if (GetMethodID() == kZSTD)
-        {
-          SetLevel_zstd();
-        } else {
-          SetLevel_default();
-        }
         SetDictionary();
         SetOrder();
         SetSolidBlockSize();
@@ -1018,10 +1043,13 @@ void CCompressDialog::SetNearestSelectComboBox(NControl::CComboBox &comboBox, UI
     comboBox.SetCurSel(0);
 }
 
-void CCompressDialog::SetLevel_zstd()
+void CCompressDialog::SetLevel()
 {
   UInt32 level = GetLevel2();
+  UInt32 LevelsMask;
+  UInt32 langID = 0;
 
+  SetMethod(GetMethodID());
   const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
   {
     int index = FindRegistryFormat(ai.Name);
@@ -1030,109 +1058,50 @@ void CCompressDialog::SetLevel_zstd()
       const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
       if (fo.Level <= 22)
         level = fo.Level;
+      else
+        level = 5;
     }
   }
 
-  /* ZStandard has 22 levels */
   m_Level.ResetContent();
+  if (GetMethodID() == kZSTD)
+    LevelsMask = g_Formats[6].LevelsMask;
+  else if (GetMethodID() == kLZ4)
+    LevelsMask = g_Formats[7].LevelsMask;
+  else if (GetMethodID() == kLZ5)
+    LevelsMask = g_Formats[8].LevelsMask;
+  else
+    LevelsMask = g_Formats[GetStaticFormatIndex()].LevelsMask;
+
   for (unsigned i = 0; i <= 22; i++)
   {
     TCHAR s[40];
     TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 };
-    ConvertUInt32ToString(i, s);
-    lstrcat(t, s);
-    switch (i) {
-    case 0:
-      lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(IDS_METHOD_STORE));
-      lstrcat(t, TEXT(")"));
-      break;
-    case 1:
-      lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(IDS_METHOD_FASTEST));
-      lstrcat(t, TEXT(")"));
-      break;
-    case 5:
-      lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(IDS_METHOD_FAST));
-      lstrcat(t, TEXT(")"));
-      break;
-    case 11:
-      lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(IDS_METHOD_NORMAL));
-      lstrcat(t, TEXT(")"));
-      break;
-    case 17:
-      lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(IDS_METHOD_MAXIMUM));
-      lstrcat(t, TEXT(")"));
-      break;
-    case 22:
-      lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(IDS_METHOD_ULTRA));
-      lstrcat(t, TEXT(")"));
-      break;
-    }
-    int index = (int)m_Level.AddString(t);
-    m_Level.SetItemData(index, i);
-  }
 
-  SetNearestSelectComboBox(m_Level, level);
+    // max reached
-  return;
+    if (LevelsMask < (UInt32)(1 << i))
-}
+      break;
 
-void CCompressDialog::SetLevel_default()
+    if ((LevelsMask & (1 << i)) != 0)
-{
-  UInt32 level = GetLevel2();
-
-  const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
     {
-    int index = FindRegistryFormat(ai.Name);
-    if (index >= 0)
-    {
-      const NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
-      if (fo.Level <= 9)
-        level = fo.Level;
-      else
-        level = 9;
-    }
-  }
-
-  /* 9 default levels */
-  m_Level.ResetContent();
-  const CFormatInfo &fi = g_Formats[GetStaticFormatIndex()];
-
-  for (unsigned i = 0; i <= 9; i++)
-  {
-    if ((fi.LevelsMask & (1 << i)) != 0)
-    {
-      TCHAR s[40];
-      TCHAR t[50] = { TEXT('L'), TEXT('e'), TEXT('v'), TEXT('e'), TEXT('l'), TEXT(' '), 0 };
-      UInt32 langID = g_Levels[i];
       ConvertUInt32ToString(i, s);
       lstrcat(t, s);
       lstrcat(t, TEXT(" ("));
-      lstrcat(t, LangString(langID));
+      lstrcat(t, LangString(g_Levels[langID]));
       lstrcat(t, TEXT(")"));
       int index = (int)m_Level.AddString(t);
       m_Level.SetItemData(index, i);
-    }
+      langID++;
-  }
-
-  SetNearestSelectComboBox(m_Level, level);
-  return;
-}
-
-void CCompressDialog::SetLevel()
-{
-  SetMethod();
-
-  if (GetMethodID() == kZSTD)
-  {
-    SetLevel_zstd();
     } else {
-    SetLevel_default();
+      ConvertUInt32ToString(i, s);
+      lstrcat(t, s);
+      int index = (int)m_Level.AddString(t);
+      m_Level.SetItemData(index, i);
     }
+  }
+
+  SetNearestSelectComboBox(m_Level, level);
+  return;
 }
 
 void CCompressDialog::SetMethod(int keepMethodId)
@@ -1645,6 +1614,9 @@ void CCompressDialog::SetNumThreads()
   int methodID = GetMethodID();
   switch (methodID)
   {
+    case kLZ4: numAlgoThreadsMax = 128; break;
+    case kLZ5: numAlgoThreadsMax = 128; break;
+    case kZSTD: numAlgoThreadsMax = 128; break;
     case kLZMA: numAlgoThreadsMax = 2; break;
     case kLZMA2: numAlgoThreadsMax = 32; break;
     case kBZip2: numAlgoThreadsMax = 32; break;
@@ -1741,38 +1713,6 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory)
       return size + decompressMemory;
     }
 
-    case kZSTD:
-    {
-      /* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=131075 */
-      size = 0;
-      switch (level) {
-        case 1: size = 824228; decompressMemory = 415024; return size;
-        case 2: size = 1282980; decompressMemory = 415024; return size;
-        case 3: size = 922532; decompressMemory = 415024; return size;
-        case 4: size = 1414052; decompressMemory = 415024; return size;
-        case 5: size = 1545124; decompressMemory = 415024; return size;
-        case 6: size = 1807268; decompressMemory = 415024; return size;
-        case 7: size = 1807268; decompressMemory = 415024; return size;
-        case 8: size = 1807268; decompressMemory = 415024; return size;
-        case 9: size = 1807268; decompressMemory = 415024; return size;
-        case 10: size = 1807268; decompressMemory = 415024; return size;
-        case 11: size = 2331556; decompressMemory = 415024; return size;
-        case 12: size = 2331556; decompressMemory = 415024; return size;
-        case 13: size = 3380132; decompressMemory = 415024; return size;
-        case 14: size = 3004832; decompressMemory = 415024; return size;
-        case 15: size = 3004832; decompressMemory = 415024; return size;
-        case 16: size = 4697834; decompressMemory = 415024; return size;
-        case 17: size = 4697834; decompressMemory = 415024; return size;
-        case 18: size = 4697834; decompressMemory = 415024; return size;
-        case 19: size = 4697834; decompressMemory = 415024; return size;
-        case 20: size = 4697834; decompressMemory = 415024; return size;
-        case 21: size = 4697834; decompressMemory = 415024; return size;
-        case 22: size = 4697834; decompressMemory = 415024; return size;
-      }
-      decompressMemory = 0;
-      return size;
-    }
-
     case kDeflate:
     case kDeflate64:
     {

CPP/7zip/UI/GUI/CompressDialog.h

@@ -130,8 +130,6 @@ class CCompressDialog: public NWindows::NControl::CModalDialog
   void SetNearestSelectComboBox(NWindows::NControl::CComboBox &comboBox, UInt32 value);
 
   void SetLevel();
-  void SetLevel_zstd();
-  void SetLevel_default();
   
   void SetMethod(int keepMethodId = -1);
   int GetMethodID();