xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-06 15:14:59 -06:00
Code Issues Packages Projects Releases Wiki Activity

Add Fast LZMA2 codec

Browse Source
This commit is contained in:
conor42
2018-11-05 21:22:10 +10:00
parent ed069bab9e
commit ab10047253
41 changed files with 9060 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.gitignore vendored Normal file
View File

@@ -0,0 +1,8 @@
*.obj
*.exe
*.res
*.dll
*.pch
*.exp
*.lib

82
C/fast-lzma2/atomic.h Normal file
View File

@@ -0,0 +1,82 @@
/*
* Copyright (c) 2018 Conor McCarthy
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*
*/
#ifndef FL2_ATOMIC_H
#define FL2_ATOMIC_H
#if defined (__cplusplus)
extern "C" {
#endif
/* atomic add */
#if !defined(FL2_SINGLETHREAD) && defined(_WIN32)
#ifdef WINVER
#undef WINVER
#endif
#define WINVER 0x0600
#ifdef _WIN32_WINNT
#undef _WIN32_WINNT
#endif
#define _WIN32_WINNT 0x0600
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
typedef LONG volatile FL2_atomic;
#define ATOMIC_INITIAL_VALUE -1
#define FL2_atomic_increment(n) InterlockedIncrement(&n)
#define FL2_atomic_add(n, a) InterlockedAdd(&n, a)
#define FL2_nonAtomic_increment(n) (++n)
#elif !defined(FL2_SINGLETHREAD) && defined(__GNUC__)
typedef long FL2_atomic;
#define ATOMIC_INITIAL_VALUE 0
#define FL2_atomic_increment(n) __sync_fetch_and_add(&n, 1)
#define FL2_atomic_add(n, a) __sync_fetch_and_add(&n, a)
#define FL2_nonAtomic_increment(n) (n++)
#elif !defined(FL2_SINGLETHREAD) && defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) /* C11 */
#include <stdatomic.h>
typedef _Atomic long FL2_atomic;
#define ATOMIC_INITIAL_VALUE 0
#define FL2_atomic_increment(n) atomic_fetch_add(&n, 1)
#define FL2_atomic_add(n, a) atomic_fetch_add(&n, a)
#define FL2_nonAtomic_increment(n) (n++)
#else /* No atomics */
# ifndef FL2_SINGLETHREAD
# error No atomic operations available. Change compiler config or define FL2_SINGLETHREAD for the entire build.
# endif
typedef long FL2_atomic;
#define ATOMIC_INITIAL_VALUE 0
#define FL2_atomic_increment(n) (n++)
#define FL2_atomic_add(n, a) (n += (a))
#define FL2_nonAtomic_increment(n) (n++)
#endif /* FL2_SINGLETHREAD */
#if defined (__cplusplus)
}
#endif
#endif /* FL2_ATOMIC_H */

86
C/fast-lzma2/compiler.h Normal file
View File

@@ -0,0 +1,86 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_COMPILER_H
#define ZSTD_COMPILER_H
/*-*******************************************************
* Compiler specifics
*********************************************************/
/* force inlining */
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# define INLINE_KEYWORD inline
#else
# define INLINE_KEYWORD
#endif
#if defined(__GNUC__)
# define FORCE_INLINE_ATTR __attribute__((always_inline))
#elif defined(_MSC_VER)
# define FORCE_INLINE_ATTR __forceinline
#else
# define FORCE_INLINE_ATTR
#endif
/**
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
* parameters. They must be inlined for the compiler to eliminate the constant
* branches.
*/
#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR
/**
* HINT_INLINE is used to help the compiler generate better code. It is *not*
* used for "templates", so it can be tweaked based on the compilers
* performance.
*
* gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the
* always_inline attribute.
*
* clang up to 5.0.0 (trunk) benefit tremendously from the always_inline
* attribute.
*/
#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5
# define HINT_INLINE static INLINE_KEYWORD
#else
# define HINT_INLINE static INLINE_KEYWORD FORCE_INLINE_ATTR
#endif
/* force no inlining */
#ifdef _MSC_VER
# define FORCE_NOINLINE static __declspec(noinline)
#else
# ifdef __GNUC__
# define FORCE_NOINLINE static __attribute__((__noinline__))
# else
# define FORCE_NOINLINE static
# endif
#endif
/* prefetch */
#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0)
#elif defined(__GNUC__)
# define PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0)
#else
# define PREFETCH(ptr) /* disabled */
#endif
/* disable warnings */
#ifdef _MSC_VER /* Visual Studio */
# include <intrin.h> /* For Visual 2005 */
# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */
# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */
# pragma warning(disable : 4324) /* disable: C4324: padded structure */
#endif
#endif /* ZSTD_COMPILER_H */

115
C/fast-lzma2/count.h Normal file
View File

@@ -0,0 +1,115 @@
#ifndef ZSTD_COUNT_H_
#define ZSTD_COUNT_H_
#include "mem.h"
#if defined (__cplusplus)
extern "C" {
#endif
/*-*************************************
* Match length counter
***************************************/
static unsigned ZSTD_NbCommonBytes(register size_t val)
{
if (MEM_isLittleEndian()) {
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanForward64(&r, (U64)val);
return (unsigned)(r >> 3);
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (__builtin_ctzll((U64)val) >> 3);
# else
static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2,
0, 3, 1, 3, 1, 4, 2, 7,
0, 2, 3, 6, 1, 5, 3, 5,
1, 3, 4, 4, 2, 5, 6, 7,
7, 0, 1, 2, 3, 3, 4, 6,
2, 6, 5, 5, 3, 4, 5, 6,
7, 1, 2, 4, 6, 4, 4, 5,
7, 2, 6, 5, 7, 6, 7, 7 };
return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58];
# endif
}
else { /* 32 bits */
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanForward(&r, (U32)val);
return (unsigned)(r >> 3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_ctz((U32)val) >> 3);
# else
static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0,
3, 2, 2, 1, 3, 2, 0, 1,
3, 3, 1, 2, 2, 2, 2, 0,
3, 1, 2, 0, 1, 0, 1, 1 };
return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27];
# endif
}
}
else { /* Big Endian CPU */
if (MEM_64bits()) {
# if defined(_MSC_VER) && defined(_WIN64)
unsigned long r = 0;
_BitScanReverse64(&r, val);
return (unsigned)(r >> 3);
# elif defined(__GNUC__) && (__GNUC__ >= 4)
return (__builtin_clzll(val) >> 3);
# else
unsigned r;
const unsigned n32 = sizeof(size_t) * 4; /* calculate this way due to compiler complaining in 32-bits mode */
if (!(val >> n32)) { r = 4; }
else { r = 0; val >>= n32; }
if (!(val >> 16)) { r += 2; val >>= 8; }
else { val >>= 24; }
r += (!val);
return r;
# endif
}
else { /* 32 bits */
# if defined(_MSC_VER)
unsigned long r = 0;
_BitScanReverse(&r, (unsigned long)val);
return (unsigned)(r >> 3);
# elif defined(__GNUC__) && (__GNUC__ >= 3)
return (__builtin_clz((U32)val) >> 3);
# else
unsigned r;
if (!(val >> 16)) { r = 2; val >>= 8; }
else { r = 0; val >>= 24; }
r += (!val);
return r;
# endif
}
}
}
MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit)
{
const BYTE* const pStart = pIn;
const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t) - 1);
if (pIn < pInLoopLimit) {
{ size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (diff) return ZSTD_NbCommonBytes(diff); }
pIn += sizeof(size_t); pMatch += sizeof(size_t);
while (pIn < pInLoopLimit) {
size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn);
if (!diff) { pIn += sizeof(size_t); pMatch += sizeof(size_t); continue; }
pIn += ZSTD_NbCommonBytes(diff);
return (size_t)(pIn - pStart);
}
}
if (MEM_64bits() && (pIn<(pInLimit - 3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn += 4; pMatch += 4; }
if ((pIn<(pInLimit - 1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn += 2; pMatch += 2; }
if ((pIn<pInLimit) && (*pMatch == *pIn)) pIn++;
return (size_t)(pIn - pStart);
}
#if defined (__cplusplus)
}
#endif
#endif /* ZSTD_COUNT_H_ */

20
C/fast-lzma2/data_block.h Normal file
View File

@@ -0,0 +1,20 @@
#include "mem.h"
#ifndef FL2_DATA_BLOCK_H_
#define FL2_DATA_BLOCK_H_
#if defined (__cplusplus)
extern "C" {
#endif
typedef struct {
const BYTE* data;
size_t start;
size_t end;
} FL2_dataBlock;
#if defined (__cplusplus)
}
#endif
#endif /* FL2_DATA_BLOCK_H_ */

455
C/fast-lzma2/fast-lzma2.h Normal file
View File

@@ -0,0 +1,455 @@
/*
* Copyright (c) 2017-present, Conor McCarthy
* All rights reserved.
* Based on zstd.h copyright Yann Collet
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#if defined (__cplusplus)
extern "C" {
#endif
#ifndef FAST_LZMA2_H
#define FAST_LZMA2_H
/* ====== Dependency ======*/
#include <stddef.h> /* size_t */
/* ===== FL2LIB_API : control library symbols visibility ===== */
#ifndef FL2LIB_VISIBILITY
# if defined(__GNUC__) && (__GNUC__ >= 4)
# define FL2LIB_VISIBILITY __attribute__ ((visibility ("default")))
# else
# define FL2LIB_VISIBILITY
# endif
#endif
#if defined(FL2_DLL_EXPORT) && (FL2_DLL_EXPORT==1)
# define FL2LIB_API __declspec(dllexport) FL2LIB_VISIBILITY
#elif defined(FL2_DLL_IMPORT) && (FL2_DLL_IMPORT==1)
# define FL2LIB_API __declspec(dllimport) FL2LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/
#else
# define FL2LIB_API FL2LIB_VISIBILITY
#endif
/* ====== Calling convention ======*/
#if !defined _WIN32 || defined __x86_64__s || defined _M_X64 || (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8)
# define FL2LIB_CALL
#elif defined(__GNUC__)
# define FL2LIB_CALL __attribute__((cdecl))
#elif defined(_MSC_VER)
# define FL2LIB_CALL __cdecl
#else
# define FL2LIB_CALL
#endif
/*******************************************************************************************************
Introduction
*********************************************************************************************************/
/*------ Version ------*/
#define FL2_VERSION_MAJOR 0
#define FL2_VERSION_MINOR 9
#define FL2_VERSION_RELEASE 2
#define FL2_VERSION_NUMBER (FL2_VERSION_MAJOR *100*100 + FL2_VERSION_MINOR *100 + FL2_VERSION_RELEASE)
FL2LIB_API unsigned FL2LIB_CALL FL2_versionNumber(void); /**< useful to check dll version */
#define FL2_LIB_VERSION FL2_VERSION_MAJOR.FL2_VERSION_MINOR.FL2_VERSION_RELEASE
#define FL2_QUOTE(str) #str
#define FL2_EXPAND_AND_QUOTE(str) FL2_QUOTE(str)
#define FL2_VERSION_STRING FL2_EXPAND_AND_QUOTE(FL2_LIB_VERSION)
FL2LIB_API const char* FL2LIB_CALL FL2_versionString(void);
/***************************************
* Simple API
***************************************/
#define FL2_MAXTHREADS 200
/*! FL2_compress() :
* Compresses `src` content as a single LZMA2 compressed stream into already allocated `dst`.
* Call FL2_compressMt() to use > 1 thread. Specify nbThreads = 0 to use all cores.
* @return : compressed size written into `dst` (<= `dstCapacity),
* or an error code if it fails (which can be tested using FL2_isError()). */
FL2LIB_API size_t FL2LIB_CALL FL2_compress(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
FL2LIB_API size_t FL2LIB_CALL FL2_compressMt(void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel,
unsigned nbThreads);
/*! FL2_decompress() :
* `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames.
* `dstCapacity` is an upper bound of originalSize to regenerate.
* If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data.
* @return : the number of bytes decompressed into `dst` (<= `dstCapacity`),
* or an errorCode if it fails (which can be tested using FL2_isError()). */
FL2LIB_API size_t FL2LIB_CALL FL2_decompress(void* dst, size_t dstCapacity,
const void* src, size_t compressedSize);
/*! FL2_findDecompressedSize()
* `src` should point to the start of a LZMA2 encoded stream.
* `srcSize` must be at least as large as the LZMA2 stream including end marker.
* @return : - decompressed size of the stream in `src`, if known
* - FL2_CONTENTSIZE_ERROR if an error occurred (e.g. corruption, srcSize too small)
* note 1 : a 0 return value means the frame is valid but "empty".
* note 2 : decompressed size can be very large (64-bits value),
* potentially larger than what local system can handle as a single memory segment.
* In which case, it's necessary to use streaming mode to decompress data.
* note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified.
* Always ensure return value fits within application's authorized limits.
* Each application can set its own limits. */
#define FL2_CONTENTSIZE_ERROR (size_t)-1
FL2LIB_API size_t FL2LIB_CALL FL2_findDecompressedSize(const void *src, size_t srcSize);
/*====== Helper functions ======*/
#define FL2_COMPRESSBOUND(srcSize) ((srcSize) + (((srcSize) + 0xFFF) / 0x1000) * 3 + 6) /* this formula calculates the maximum size of data stored in uncompressed chunks */
FL2LIB_API size_t FL2LIB_CALL FL2_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */
FL2LIB_API unsigned FL2LIB_CALL FL2_isError(size_t code); /*!< tells if a `size_t` function result is an error code */
FL2LIB_API const char* FL2LIB_CALL FL2_getErrorName(size_t code); /*!< provides readable string from an error code */
FL2LIB_API int FL2LIB_CALL FL2_maxCLevel(void); /*!< maximum compression level available */
FL2LIB_API int FL2LIB_CALL FL2_maxHighCLevel(void); /*!< maximum compression level available in high mode */
/***************************************
* Explicit memory management
***************************************/
/*= Compression context
* When compressing many times,
* it is recommended to allocate a context just once, and re-use it for each successive compression operation.
* This will make workload friendlier for system's memory.
* The context may not use the number of threads requested if the library is compiled for single-threaded
* compression or nbThreads > FL2_MAXTHREADS. Call FL2_CCtx_nbThreads to obtain the actual number. */
typedef struct FL2_CCtx_s FL2_CCtx;
FL2LIB_API FL2_CCtx* FL2LIB_CALL FL2_createCCtx(void);
FL2LIB_API FL2_CCtx* FL2LIB_CALL FL2_createCCtxMt(unsigned nbThreads);
FL2LIB_API void FL2LIB_CALL FL2_freeCCtx(FL2_CCtx* cctx);
FL2LIB_API unsigned FL2LIB_CALL FL2_CCtx_nbThreads(const FL2_CCtx* ctx);
/*! FL2_compressCCtx() :
* Same as FL2_compress(), requires an allocated FL2_CCtx (see FL2_createCCtx()). */
FL2LIB_API size_t FL2LIB_CALL FL2_compressCCtx(FL2_CCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
int compressionLevel);
/************************************************
* Caller-managed data buffer and overlap section
************************************************/
typedef struct {
unsigned char *data;
size_t start; /* start = 0 (first block) or overlap */
size_t end; /* never < overlap */
size_t bufSize; /* allocation size */
} FL2_blockBuffer;
typedef int (FL2LIB_CALL *FL2_progressFn)(size_t done, void* opaque);
/* Get the size of the overlap section. */
FL2LIB_API size_t FL2LIB_CALL FL2_blockOverlap(const FL2_CCtx* ctx);
/* Copy the overlap section to the start to prepare for more data */
FL2LIB_API void FL2LIB_CALL FL2_shiftBlock(FL2_CCtx* ctx, FL2_blockBuffer *block);
/* Copy the overlap to a different buffer. This allows a dual-buffer configuration where
* data is read into one block while the other is compressed. */
FL2LIB_API void FL2LIB_CALL FL2_shiftBlock_switch(FL2_CCtx* ctx, FL2_blockBuffer *block, unsigned char *dst);
FL2LIB_API void FL2LIB_CALL FL2_beginFrame(FL2_CCtx* const cctx);
/*! FL2_compressCCtxBlock() :
* Same as FL2_compressCCtx except the caller is responsible for supplying an overlap section.
* The FL2_p_overlapFraction parameter will not be used.
* srcStart + srcSize should equal the dictionary size except on the last call.
* Can be called multiple times. FL2_endFrame() must be called when finished.
* For compatibility with this library the caller must write a property byte at
* the beginning of the output. Obtain it by calling FL2_dictSizeProp() before
* compressing the first block or after the last. No hash will be written, but
* the caller can calculate it using the interface in xxhash.h, write it at the end,
* and set bit 7 in the property byte. */
FL2LIB_API size_t FL2LIB_CALL FL2_compressCCtxBlock(FL2_CCtx* ctx,
void* dst, size_t dstCapacity,
const FL2_blockBuffer *block,
FL2_progressFn progress, void* opaque);
/*! FL2_endFrame() :
* Write the end marker to terminate the LZMA2 stream.
* Must be called after compressing with FL2_compressCCtxBlock() */
FL2LIB_API size_t FL2LIB_CALL FL2_endFrame(FL2_CCtx* ctx,
void* dst, size_t dstCapacity);
typedef int (FL2LIB_CALL *FL2_writerFn)(const void* src, size_t srcSize, void* opaque);
/*! FL2_compressCCtxBlock_toFn() :
* Same as FL2_compressCCtx except the caller is responsible for supplying an
* overlap section, and compressed data is written to a callback function.
* The FL2_p_overlapFraction parameter will not be used.
* Can be called multiple times. FL2_endFrame_toFn() must be called when finished. */
FL2LIB_API size_t FL2LIB_CALL FL2_compressCCtxBlock_toFn(FL2_CCtx* ctx,
FL2_writerFn writeFn, void* opaque,
const FL2_blockBuffer *block,
FL2_progressFn progress);
/*! FL2_endFrame() :
* Write the end marker to a callback function to terminate the LZMA2 stream.
* Must be called after compressing with FL2_compressCCtxBlock_toFn() */
FL2LIB_API size_t FL2LIB_CALL FL2_endFrame_toFn(FL2_CCtx* ctx,
FL2_writerFn writeFn, void* opaque);
/*! FL2_dictSizeProp() :
* Get the dictionary size property.
* Intended for use with the FL2_p_omitProperties parameter for creating a
* 7-zip compatible LZMA2 stream. */
FL2LIB_API unsigned char FL2LIB_CALL FL2_dictSizeProp(FL2_CCtx* ctx);
/*= Decompression context
* When decompressing many times,
* it is recommended to allocate a context only once,
* and re-use it for each successive compression operation.
* This will make the workload friendlier for the system's memory.
* Use one context per thread for parallel execution. */
typedef struct CLzma2Dec_s FL2_DCtx;
FL2LIB_API FL2_DCtx* FL2LIB_CALL FL2_createDCtx(void);
FL2LIB_API size_t FL2LIB_CALL FL2_freeDCtx(FL2_DCtx* dctx);
/*! FL2_decompressDCtx() :
* Same as FL2_decompress(), requires an allocated FL2_DCtx (see FL2_createDCtx()) */
FL2LIB_API size_t FL2LIB_CALL FL2_decompressDCtx(FL2_DCtx* ctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize);
/****************************
* Streaming
****************************/
typedef struct FL2_inBuffer_s {
const void* src; /**< start of input buffer */
size_t size; /**< size of input buffer */
size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */
} FL2_inBuffer;
typedef struct FL2_outBuffer_s {
void* dst; /**< start of output buffer */
size_t size; /**< size of output buffer */
size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */
} FL2_outBuffer;
/*-***********************************************************************
* Streaming compression - HowTo
*
* A FL2_CStream object is required to track streaming operation.
* Use FL2_createCStream() and FL2_freeCStream() to create/release resources.
* FL2_CStream objects can be reused multiple times on consecutive compression operations.
* It is recommended to re-use FL2_CStream in situations where many streaming operations will be achieved consecutively,
* since it will play nicer with system's memory, by re-using already allocated memory.
*
* Start a new compression by initializing FL2_CStream.
* Use FL2_initCStream() to start a new compression operation.
*
* Use FL2_compressStream() repetitively to consume input stream.
* The function will automatically update both `pos` fields.
* It will always consume the entire input unless an error occurs,
* unlike the decompression function.
* @return : a size hint - remaining capacity to fill before compression occurs,
* or an error code, which can be tested using FL2_isError().
* Note : it's just a hint, any other value will work fine.
*
* At any moment, it's possible, but not recommended, to flush whatever data remains
* within internal buffer using FL2_flushStream().
* `output->pos` will be updated.
* Note 1 : this will reduce compression ratio because the algorithm is block-based.
* Note 2 : some content might still be left within internal buffers if `output->size` is too small.
* @return : nb of bytes still present within internal buffers (0 if they're empty)
* or an error code, which can be tested using FL2_isError().
*
* FL2_endStream() instructs to finish a frame.
* It will perform a flush and write the LZMA2 termination byte (required).
* FL2_endStream() may not be able to flush full data if `output->size` is too small.
* In which case, call again FL2_endStream() to complete the flush.
* @return : 0 if stream fully completed and flushed,
* or >0 to indicate the nb of bytes still present within the internal buffers,
* or an error code, which can be tested using FL2_isError().
*
* *******************************************************************/
typedef struct FL2_CStream_s FL2_CStream;
/*===== FL2_CStream management functions =====*/
FL2LIB_API FL2_CStream* FL2LIB_CALL FL2_createCStream(void);
FL2LIB_API size_t FL2LIB_CALL FL2_freeCStream(FL2_CStream* fcs);
/*===== Streaming compression functions =====*/
FL2LIB_API size_t FL2LIB_CALL FL2_initCStream(FL2_CStream* fcs, int compressionLevel);
FL2LIB_API size_t FL2LIB_CALL FL2_compressStream(FL2_CStream* fcs, FL2_outBuffer* output, FL2_inBuffer* input);
FL2LIB_API size_t FL2LIB_CALL FL2_flushStream(FL2_CStream* fcs, FL2_outBuffer* output);
FL2LIB_API size_t FL2LIB_CALL FL2_endStream(FL2_CStream* fcs, FL2_outBuffer* output);
/*-***************************************************************************
* Streaming decompression - HowTo
*
* A FL2_DStream object is required to track streaming operations.
* Use FL2_createDStream() and FL2_freeDStream() to create/release resources.
* FL2_DStream objects can be re-used multiple times.
*
* Use FL2_initDStream() to start a new decompression operation.
* @return : recommended first input size
*
* Use FL2_decompressStream() repetitively to consume your input.
* The function will update both `pos` fields.
* If `input.pos < input.size`, some input has not been consumed.
* It's up to the caller to present again remaining data.
* More data must be loaded if `input.pos + LZMA_REQUIRED_INPUT_MAX >= input.size`
* If `output.pos < output.size`, decoder has flushed everything it could.
* @return : 0 when a frame is completely decoded and fully flushed,
* an error code, which can be tested using FL2_isError(),
* 1, which means there is still some decoding to do to complete current frame.
* *******************************************************************************/
#define LZMA_REQUIRED_INPUT_MAX 20
typedef struct FL2_DStream_s FL2_DStream;
/*===== FL2_DStream management functions =====*/
FL2LIB_API FL2_DStream* FL2LIB_CALL FL2_createDStream(void);
FL2LIB_API size_t FL2LIB_CALL FL2_freeDStream(FL2_DStream* fds);
/*===== Streaming decompression functions =====*/
FL2LIB_API size_t FL2LIB_CALL FL2_initDStream(FL2_DStream* fds);
FL2LIB_API size_t FL2LIB_CALL FL2_decompressStream(FL2_DStream* fds, FL2_outBuffer* output, FL2_inBuffer* input);
/*-***************************************************************************
* Compression parameters - HowTo
*
* Any function that takes a 'compressionLevel' parameter will replace any
* parameters affected by compression level that are already set.
* Call FL2_CCtx_setParameter with FL2_p_compressionLevel to set the level,
* then call FL2_CCtx_setParameter again with any other settings to change.
* Specify compressionLevel=0 when calling a compression function.
* *******************************************************************************/
#define FL2_DICTLOG_MAX_32 27
#define FL2_DICTLOG_MAX_64 30
#define FL2_DICTLOG_MAX ((unsigned)(sizeof(size_t) == 4 ? FL2_DICTLOG_MAX_32 : FL2_DICTLOG_MAX_64))
#define FL2_DICTLOG_MIN 20
#define FL2_CHAINLOG_MAX 14
#define FL2_CHAINLOG_MIN 4
#define FL2_SEARCHLOG_MAX (FL2_CHAINLOG_MAX-1)
#define FL2_SEARCHLOG_MIN 0
#define FL2_FASTLENGTH_MIN 6 /* only used by optimizer */
#define FL2_FASTLENGTH_MAX 273 /* only used by optimizer */
#define FL2_BLOCK_OVERLAP_MIN 0
#define FL2_BLOCK_OVERLAP_MAX 14
#define FL2_BLOCK_LOG_MIN 12
#define FL2_BLOCK_LOG_MAX 32
#define FL2_SEARCH_DEPTH_MIN 6
#define FL2_SEARCH_DEPTH_MAX 254
#define FL2_BUFFER_SIZE_LOG_MIN 6
#define FL2_BUFFER_SIZE_LOG_MAX 12
#define FL2_LC_MIN 0
#define FL2_LC_MAX 4
#define FL2_LP_MIN 0
#define FL2_LP_MAX 4
#define FL2_PB_MIN 0
#define FL2_PB_MAX 4
typedef enum {
/* compression parameters */
FL2_p_compressionLevel, /* Update all compression parameters according to pre-defined cLevel table
* Default level is FL2_CLEVEL_DEFAULT==9.
* Setting FL2_p_highCompression to 1 switches to an alternate cLevel table.
* Special: value 0 means "do not change cLevel". */
FL2_p_highCompression, /* Maximize compression ratio for a given dictionary size.
* Has 9 levels instead of 12, with dictionaryLog 20 - 28. */
FL2_p_7zLevel, /* For use by the 7-zip fork employing this library. 1 - 9 */
FL2_p_dictionaryLog, /* Maximum allowed back-reference distance, expressed as power of 2.
* Must be clamped between FL2_DICTLOG_MIN and FL2_DICTLOG_MAX.
* Special: value 0 means "do not change dictionaryLog". */
FL2_p_overlapFraction, /* The radix match finder is block-based, so some overlap is retained from
* each block to improve compression of the next. This value is expressed
* as n / 16 of the block size (dictionary size). Larger values are slower.
* Values above 2 mostly yield only a small improvement in compression. */
FL2_p_blockSize,
FL2_p_bufferLog, /* Buffering speeds up the matchfinder. Buffer size is
* 2 ^ (dictionaryLog - bufferLog). Lower number = slower, better compression,
* higher memory usage. */
FL2_p_chainLog, /* Size of the full-search table, as a power of 2.
* Resulting table size is (1 << (chainLog+2)).
* Larger tables result in better and slower compression.
* This parameter is useless when using "fast" strategy.
* Special: value 0 means "do not change chainLog". */
FL2_p_searchLog, /* Number of search attempts, as a power of 2, made by the HC3 match finder
* used only in hybrid mode.
* More attempts result in slightly better and slower compression.
* This parameter is not used by the "fast" and "optimize" strategies.
* Special: value 0 means "do not change searchLog". */
FL2_p_literalCtxBits, /* lc value for LZMA2 encoder */
FL2_p_literalPosBits, /* lp value for LZMA2 encoder */
FL2_p_posBits, /* pb value for LZMA2 encoder */
FL2_p_searchDepth, /* Match finder will resolve string matches up to this length. If a longer
* match exists further back in the input, it will not be found. */
FL2_p_fastLength, /* Only useful for strategies >= opt.
* Length of Match considered "good enough" to stop search.
* Larger values make compression stronger and slower.
* Special: value 0 means "do not change fastLength". */
FL2_p_divideAndConquer, /* Split long chains of 2-byte matches into shorter chains with a small overlap
* during further processing. Allows buffering of all chains at length 2.
* Faster, less compression. Generally a good tradeoff. Enabled by default. */
FL2_p_strategy, /* 1 = fast; 2 = optimize, 3 = ultra (hybrid mode).
* The higher the value of the selected strategy, the more complex it is,
* resulting in stronger and slower compression.
* Special: value 0 means "do not change strategy". */
#ifndef NO_XXHASH
FL2_p_doXXHash, /* Calculate a 32-bit xxhash value from the input data and store it
* after the stream terminator. The value will be checked on decompression.
* 0 = do not calculate; 1 = calculate (default) */
#endif
FL2_p_omitProperties, /* Omit the property byte at the start of the stream. For use within 7-zip */
/* or other containers which store the property byte elsewhere. */
/* Cannot be decoded by this library. */
#ifdef RMF_REFERENCE
FL2_p_useReferenceMF /* Use the reference matchfinder for development purposes. SLOW. */
#endif
} FL2_cParameter;
/*! FL2_CCtx_setParameter() :
* Set one compression parameter, selected by enum FL2_cParameter.
* @result : informational value (typically, the one being set, possibly corrected),
* or an error code (which can be tested with FL2_isError()). */
FL2LIB_API size_t FL2LIB_CALL FL2_CCtx_setParameter(FL2_CCtx* cctx, FL2_cParameter param, unsigned value);
FL2LIB_API size_t FL2LIB_CALL FL2_CStream_setParameter(FL2_CStream* fcs, FL2_cParameter param, unsigned value);
/***************************************
* Context memory usage
***************************************/
/*! FL2_estimate*() :
* These functions estimate memory usage of a CCtx before its creation or before any operation has begun.
* FL2_estimateCCtxSize() will provide a budget large enough for any compression level up to selected one.
* To use FL2_estimateCCtxSize_usingCCtx, set the compression level and any other settings for the context,
* then call the function. Some allocation occurs when the context is created, but the large memory buffers
* used for string matching are allocated only when compression begins. */
FL2LIB_API size_t FL2LIB_CALL FL2_estimateCCtxSize(int compressionLevel, unsigned nbThreads); /*!< memory usage determined by level */
FL2LIB_API size_t FL2LIB_CALL FL2_estimateCCtxSize_usingCCtx(const FL2_CCtx* cctx); /*!< memory usage determined by settings */
FL2LIB_API size_t FL2LIB_CALL FL2_estimateCStreamSize(int compressionLevel, unsigned nbThreads);
FL2LIB_API size_t FL2LIB_CALL FL2_estimateCStreamSize_usingCCtx(const FL2_CStream* fcs);
#endif /* FAST_LZMA2_H */
#if defined (__cplusplus)
}
#endif

262
C/fast-lzma2/fastpos_table.h Normal file
View File

@@ -0,0 +1,262 @@
/* This file has been automatically generated by fastpos_tablegen.c. */
/* Copied from the XZ project */
static const BYTE distance_table[1 << kFastDistBits] = {
0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23
};

61
C/fast-lzma2/fl2_common.c Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/*-*************************************
* Dependencies
***************************************/
#include <stdlib.h> /* malloc, calloc, free */
#include <string.h> /* memset */
#include "fast-lzma2.h"
#include "fl2_error_private.h"
#include "fl2_internal.h"
/*-****************************************
* Version
******************************************/
FL2LIB_API unsigned FL2LIB_CALL FL2_versionNumber(void) { return FL2_VERSION_NUMBER; }
FL2LIB_API const char* FL2LIB_CALL FL2_versionString(void) { return FL2_VERSION_STRING; }
FL2LIB_API size_t FL2LIB_CALL FL2_compressBound(size_t srcSize)
{
return FL2_COMPRESSBOUND(srcSize);
}
/*-****************************************
* FL2 Error Management
******************************************/
/*! FL2_isError() :
* tells if a return value is an error code */
FL2LIB_API unsigned FL2LIB_CALL FL2_isError(size_t code) { return ERR_isError(code); }
/*! FL2_getErrorName() :
* provides error code string from function result (useful for debugging) */
FL2LIB_API const char* FL2LIB_CALL FL2_getErrorName(size_t code) { return ERR_getErrorName(code); }
/*! FL2_getError() :
* convert a `size_t` function result into a proper FL2_errorCode enum */
FL2LIB_API FL2_ErrorCode FL2LIB_CALL FL2_getErrorCode(size_t code) { return ERR_getErrorCode(code); }
/*! FL2_getErrorString() :
* provides error code string from enum */
FL2LIB_API const char* FL2LIB_CALL FL2_getErrorString(FL2_ErrorCode code) { return ERR_getFL2ErrorString(code); }
/*! g_debuglog_enable :
* turn on/off debug traces (global switch) */
#if defined(FL2_DEBUG) && (FL2_DEBUG >= 2)
int g_debuglog_enable = 1;
#endif

1028
C/fast-lzma2/fl2_compress.c Normal file
View File

File diff suppressed because it is too large Load Diff

100
C/fast-lzma2/fl2_compress_internal.h Normal file
View File

@@ -0,0 +1,100 @@
/*
* Copyright (c) 2018, Conor McCarthy
* All rights reserved.
* Parts based on zstd_compress_internal.h copyright Yann Collet
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef FL2_COMPRESS_H
#define FL2_COMPRESS_H
/*-*************************************
* Dependencies
***************************************/
#include "mem.h"
#include "data_block.h"
#include "radix_internal.h"
#include "lzma2_enc.h"
#include "fast-lzma2.h"
#include "fl2threading.h"
#include "fl2pool.h"
#ifndef NO_XXHASH
# include "xxhash.h"
#endif
#if defined (__cplusplus)
extern "C" {
#endif
typedef struct {
unsigned dictionaryLog; /* largest match distance : larger == more compression, more memory needed during decompression; >= 27 == more memory, slower */
unsigned overlapFraction; /* overlap between consecutive blocks in 1/16 units: larger == more compression, slower */
unsigned chainLog; /* fully searched segment : larger == more compression, slower, more memory; hybrid mode only (ultra) */
unsigned searchLog; /* nb of searches : larger == more compression, slower; hybrid mode only (ultra) */
unsigned searchDepth; /* maximum depth for resolving string matches : larger == more compression, slower; >= 64 == more memory, slower */
unsigned fastLength; /* acceptable match size for parser, not less than searchDepth : larger == more compression, slower; fast bytes parameter from 7-zip */
unsigned divideAndConquer; /* split long chains of 2-byte matches into shorter chains with a small overlap : faster, somewhat less compression; enabled by default */
unsigned bufferLog; /* buffer size for processing match chains is (dictionaryLog - bufferLog) : when divideAndConquer enabled, affects compression; */
/* when divideAndConquer disabled, affects speed in a hardware-dependent manner */
FL2_strategy strategy; /* encoder strategy : fast, optimized or ultra (hybrid) */
} FL2_compressionParameters;
/*-*************************************
* Context memory management
***************************************/
typedef struct {
FL2_lzma2Parameters cParams;
RMF_parameters rParams;
unsigned compressionLevel;
BYTE highCompression;
#ifndef NO_XXHASH
BYTE doXXH;
#endif
BYTE omitProp;
} FL2_CCtx_params;
typedef struct {
FL2_CCtx* cctx;
FL2_lzmaEncoderCtx* enc;
FL2_dataBlock block;
size_t cSize;
} FL2_job;
struct FL2_CCtx_s {
FL2_CCtx_params params;
#ifndef FL2_SINGLETHREAD
FL2POOL_ctx* factory;
#endif
FL2_dataBlock curBlock;
size_t dictMax;
U64 block_total;
FL2_matchTable* matchTable;
unsigned jobCount;
FL2_job jobs[1];
};
struct FL2_CStream_s {
FL2_CCtx* cctx;
FL2_blockBuffer inBuff;
#ifndef NO_XXHASH
XXH32_state_t *xxh;
#endif
size_t thread_count;
size_t out_thread;
size_t out_pos;
size_t hash_pos;
BYTE end_marked;
BYTE wrote_prop;
};
#if defined (__cplusplus)
}
#endif
#endif /* FL2_COMPRESS_H */

35
C/fast-lzma2/fl2_error_private.c Normal file
View File

@@ -0,0 +1,35 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* The purpose of this file is to have a single list of error strings embedded in binary */
#include "fl2_error_private.h"
const char* ERR_getFL2ErrorString(ERR_enum code)
{
static const char* const notErrorCode = "Unspecified error code";
switch( code )
{
case PREFIX(no_error): return "No error detected";
case PREFIX(GENERIC): return "Error (generic)";
case PREFIX(corruption_detected): return "Corrupted block detected";
case PREFIX(checksum_wrong): return "Restored data doesn't match checksum";
case PREFIX(parameter_unsupported): return "Unsupported parameter";
case PREFIX(parameter_outOfBound): return "Parameter is out of bound";
case PREFIX(init_missing): return "Context should be init first";
case PREFIX(memory_allocation): return "Allocation error : not enough memory";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size is incorrect";
/* following error codes are not stable and may be removed or changed in a future version */
case PREFIX(maxCode):
default: return notErrorCode;
}
}

75
C/fast-lzma2/fl2_error_private.h Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* Note : this module is expected to remain private, do not expose it */
#ifndef ERROR_H_MODULE
#define ERROR_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* ****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t */
#include "fl2_errors.h" /* enum list */
/* ****************************************
* Compiler-specific
******************************************/
#if defined(__GNUC__)
# define ERR_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define ERR_STATIC static inline
#elif defined(_MSC_VER)
# define ERR_STATIC static __inline
#else
# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Customization (error_public.h)
******************************************/
typedef FL2_ErrorCode ERR_enum;
#define PREFIX(name) FL2_error_##name
/*-****************************************
* Error codes handling
******************************************/
#define FL2_ERROR(name) ((size_t)-PREFIX(name))
ERR_STATIC unsigned ERR_isError(size_t code) { return (code > FL2_ERROR(maxCode)); }
ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); }
/*-****************************************
* Error Strings
******************************************/
const char* ERR_getFL2ErrorString(ERR_enum code); /* error_private.c */
ERR_STATIC const char* ERR_getErrorName(size_t code)
{
return ERR_getFL2ErrorString(ERR_getErrorCode(code));
}
#if defined (__cplusplus)
}
#endif
#endif /* ERROR_H_MODULE */

59
C/fast-lzma2/fl2_errors.h Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef FL2_ERRORS_H_398273423
#define FL2_ERRORS_H_398273423
#if defined (__cplusplus)
extern "C" {
#endif
/*===== dependency =====*/
#include <stddef.h> /* size_t */
#include "fast-lzma2.h"
/*-****************************************
* error codes list
* note : this API is still considered unstable
* and shall not be used with a dynamic library.
* only static linking is allowed
******************************************/
typedef enum {
FL2_error_no_error = 0,
FL2_error_GENERIC = 1,
FL2_error_internal = 2,
FL2_error_corruption_detected = 3,
FL2_error_checksum_wrong = 4,
FL2_error_parameter_unsupported = 5,
FL2_error_parameter_outOfBound = 6,
FL2_error_stage_wrong = 7,
FL2_error_init_missing = 8,
FL2_error_memory_allocation = 9,
FL2_error_dstSize_tooSmall = 10,
FL2_error_srcSize_wrong = 11,
FL2_error_write_failed = 12,
FL2_error_canceled = 13,
FL2_error_maxCode = 20 /* never EVER use this value directly, it can change in future versions! Use FL2_isError() instead */
} FL2_ErrorCode;
/*! FL2_getErrorCode() :
convert a `size_t` function result into a `FL2_ErrorCode` enum type,
which can be used to compare with enum list published above */
FL2LIB_API FL2_ErrorCode FL2LIB_CALL FL2_getErrorCode(size_t functionResult);
FL2LIB_API const char* FL2LIB_CALL FL2_getErrorString(FL2_ErrorCode code); /**< Same as FL2_getErrorName, but using a `FL2_ErrorCode` enum argument */
#if defined (__cplusplus)
}
#endif
#endif /* FL2_ERRORS_H_398273423 */

113
C/fast-lzma2/fl2_internal.h Normal file
View File

@@ -0,0 +1,113 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef FL2_INTERNAL_H_
#define FL2_INTERNAL_H_
/*-*************************************
* Dependencies
***************************************/
#include "mem.h"
#include "compiler.h"
#include "fl2_error_private.h"
#if defined (__cplusplus)
extern "C" {
#endif
#define FL2_PROP_HASH_BIT 7
#define FL2_LZMA_PROP_MASK 0x3FU
#ifndef NO_XXHASH
# define XXHASH_SIZEOF sizeof(XXH32_canonical_t)
#endif
/*-*************************************
* Debug
***************************************/
#if defined(FL2_DEBUG) && (FL2_DEBUG>=1)
# include <assert.h>
#else
# ifndef assert
# define assert(condition) ((void)0)
# endif
#endif
#define FL2_STATIC_ASSERT(c) { enum { FL2_static_assert = 1/(int)(!!(c)) }; }
#if defined(FL2_DEBUG) && (FL2_DEBUG>=2)
# include <stdio.h>
extern int g_debuglog_enable;
/* recommended values for FL2_DEBUG display levels :
* 1 : no display, enables assert() only
* 2 : reserved for currently active debugging path
* 3 : events once per object lifetime (CCtx, CDict)
* 4 : events once per frame
* 5 : events once per block
* 6 : events once per sequence (*very* verbose) */
# define RAWLOG(l, ...) { \
if ((g_debuglog_enable) & (l<=FL2_DEBUG)) { \
fprintf(stderr, __VA_ARGS__); \
} }
# define DEBUGLOG(l, ...) { \
if ((g_debuglog_enable) & (l<=FL2_DEBUG)) { \
fprintf(stderr, __FILE__ ": "); \
fprintf(stderr, __VA_ARGS__); \
fprintf(stderr, " \n"); \
} }
#else
# define RAWLOG(l, ...) {} /* disabled */
# define DEBUGLOG(l, ...) {} /* disabled */
#endif
/*-*************************************
* shared macros
***************************************/
#undef MIN
#undef MAX
#define MIN(a,b) ((a)<(b) ? (a) : (b))
#define MAX(a,b) ((a)>(b) ? (a) : (b))
#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */
#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return FL2_ERROR(e); } /* check and send Error code */
MEM_STATIC U32 ZSTD_highbit32(U32 val)
{
assert(val != 0);
{
# if defined(_MSC_VER) /* Visual */
unsigned long r=0;
_BitScanReverse(&r, val);
return (unsigned)r;
# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */
return 31 - __builtin_clz(val);
# else /* Software version */
static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 };
U32 v = val;
int r;
v |= v >> 1;
v |= v >> 2;
v |= v >> 4;
v |= v >> 8;
v |= v >> 16;
r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27];
return r;
# endif
}
}
#if defined (__cplusplus)
}
#endif
#endif /* FL2_INTERNAL_H_ */

201
C/fast-lzma2/fl2pool.c Normal file
View File

@@ -0,0 +1,201 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* ====== Dependencies ======= */
#include <stddef.h> /* size_t */
#include <stdlib.h> /* malloc, calloc */
#include "fl2pool.h"
#include "fl2_internal.h"
/* ====== Compiler specifics ====== */
#if defined(_MSC_VER)
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
#ifndef FL2_SINGLETHREAD
#include "fl2threading.h" /* pthread adaptation */
/* A job is a function and an opaque argument */
typedef struct FL2POOL_job_s {
FL2POOL_function function;
void *opaque;
size_t n;
} FL2POOL_job;
struct FL2POOL_ctx_s {
/* Keep track of the threads */
ZSTD_pthread_t *threads;
size_t numThreads;
/* The queue is a single job */
FL2POOL_job queue;
/* The number of threads working on jobs */
size_t numThreadsBusy;
/* Indicates if the queue is empty */
int queueEmpty;
/* The mutex protects the queue */
ZSTD_pthread_mutex_t queueMutex;
/* Condition variable for pushers to wait on when the queue is full */
ZSTD_pthread_cond_t queuePushCond;
/* Condition variables for poppers to wait on when the queue is empty */
ZSTD_pthread_cond_t queuePopCond;
/* Indicates if the queue is shutting down */
int shutdown;
};
/* FL2POOL_thread() :
Work thread for the thread pool.
Waits for jobs and executes them.
@returns : NULL on failure else non-null.
*/
static void* FL2POOL_thread(void* opaque) {
FL2POOL_ctx* const ctx = (FL2POOL_ctx*)opaque;
if (!ctx) { return NULL; }
for (;;) {
/* Lock the mutex and wait for a non-empty queue or until shutdown */
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
while (ctx->queueEmpty && !ctx->shutdown) {
ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex);
}
/* empty => shutting down: so stop */
if (ctx->queueEmpty) {
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
return opaque;
}
/* Pop a job off the queue */
{ FL2POOL_job const job = ctx->queue;
ctx->queueEmpty = 1;
/* Unlock the mutex, signal a pusher, and run the job */
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
job.function(job.opaque, job.n);
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
ctx->numThreadsBusy--;
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
}
} /* for (;;) */
/* Unreachable */
}
FL2POOL_ctx* FL2POOL_create(size_t numThreads) {
FL2POOL_ctx* ctx;
/* Check the parameters */
if (!numThreads) { return NULL; }
/* Allocate the context and zero initialize */
ctx = (FL2POOL_ctx*)calloc(1, sizeof(FL2POOL_ctx));
if (!ctx) { return NULL; }
/* Initialize the job queue.
* It needs one extra space since one space is wasted to differentiate empty
* and full queues.
*/
ctx->numThreadsBusy = 0;
ctx->queueEmpty = 1;
(void)ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL);
(void)ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL);
(void)ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL);
ctx->shutdown = 0;
/* Allocate space for the thread handles */
ctx->threads = (ZSTD_pthread_t*)malloc(numThreads * sizeof(ZSTD_pthread_t));
ctx->numThreads = 0;
/* Check for errors */
if (!ctx->threads) { FL2POOL_free(ctx); return NULL; }
/* Initialize the threads */
{ size_t i;
for (i = 0; i < numThreads; ++i) {
if (FL2_pthread_create(&ctx->threads[i], NULL, &FL2POOL_thread, ctx)) {
ctx->numThreads = i;
FL2POOL_free(ctx);
return NULL;
} }
ctx->numThreads = numThreads;
}
return ctx;
}
/*! FL2POOL_join() :
Shutdown the queue, wake any sleeping threads, and join all of the threads.
*/
static void FL2POOL_join(FL2POOL_ctx* ctx) {
/* Shut down the queue */
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
ctx->shutdown = 1;
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
/* Wake up sleeping threads */
ZSTD_pthread_cond_broadcast(&ctx->queuePushCond);
ZSTD_pthread_cond_broadcast(&ctx->queuePopCond);
/* Join all of the threads */
{ size_t i;
for (i = 0; i < ctx->numThreads; ++i) {
FL2_pthread_join(ctx->threads[i], NULL);
} }
}
void FL2POOL_free(FL2POOL_ctx *ctx) {
if (!ctx) { return; }
FL2POOL_join(ctx);
ZSTD_pthread_mutex_destroy(&ctx->queueMutex);
ZSTD_pthread_cond_destroy(&ctx->queuePushCond);
ZSTD_pthread_cond_destroy(&ctx->queuePopCond);
free(ctx->threads);
free(ctx);
}
size_t FL2POOL_sizeof(FL2POOL_ctx *ctx) {
if (ctx==NULL) return 0; /* supports sizeof NULL */
return sizeof(*ctx)
+ ctx->numThreads * sizeof(ZSTD_pthread_t);
}
void FL2POOL_add(void* ctxVoid, FL2POOL_function function, void *opaque, size_t n) {
FL2POOL_ctx* const ctx = (FL2POOL_ctx*)ctxVoid;
if (!ctx)
return;
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
{ FL2POOL_job const job = {function, opaque, n};
/* Wait until there is space in the queue for the new job */
while (!ctx->queueEmpty && !ctx->shutdown) {
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
}
/* The queue is still going => there is space */
if (!ctx->shutdown) {
ctx->numThreadsBusy++;
ctx->queueEmpty = 0;
ctx->queue = job;
}
}
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
ZSTD_pthread_cond_signal(&ctx->queuePopCond);
}
void FL2POOL_waitAll(void *ctxVoid)
{
FL2POOL_ctx* const ctx = (FL2POOL_ctx*)ctxVoid;
if (!ctx) { return; }
ZSTD_pthread_mutex_lock(&ctx->queueMutex);
while (ctx->numThreadsBusy && !ctx->shutdown) {
ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex);
}
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
}
#endif /* FL2_SINGLETHREAD */

60
C/fast-lzma2/fl2pool.h Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
* Modified for FL2 by Conor McCarthy
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef FL2POOL_H
#define FL2POOL_H
#if defined (__cplusplus)
extern "C" {
#endif
#include <stddef.h> /* size_t */
typedef struct FL2POOL_ctx_s FL2POOL_ctx;
/*! FL2POOL_create() :
* Create a thread pool with at most `numThreads` threads.
* `numThreads` must be at least 1.
* @return : FL2POOL_ctx pointer on success, else NULL.
*/
FL2POOL_ctx *FL2POOL_create(size_t numThreads);
/*! FL2POOL_free() :
Free a thread pool returned by FL2POOL_create().
*/
void FL2POOL_free(FL2POOL_ctx *ctx);
/*! FL2POOL_sizeof() :
return memory usage of pool returned by FL2POOL_create().
*/
size_t FL2POOL_sizeof(FL2POOL_ctx *ctx);
/*! FL2POOL_function :
The function type that can be added to a thread pool.
*/
typedef void(*FL2POOL_function)(void *, size_t);
/*! FL2POOL_add() :
Add the job `function(opaque)` to the thread pool.
Possibly blocks until there is room in the queue.
Note : The function may be executed asynchronously, so `opaque` must live until the function has been completed.
*/
void FL2POOL_add(void *ctx, FL2POOL_function function, void *opaque, size_t n);
void FL2POOL_waitAll(void *ctx);
#if defined (__cplusplus)
}
#endif
#endif

75
C/fast-lzma2/fl2threading.c Normal file
View File

@@ -0,0 +1,75 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
/**
* This file will hold wrapper for systems, which do not support pthreads
*/
/* create fake symbol to avoid empty translation unit warning */
int g_ZSTD_threading_useles_symbol;
#if !defined(FL2_SINGLETHREAD) && defined(_WIN32)
/**
* Windows minimalist Pthread Wrapper, based on :
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
*/
/* === Dependencies === */
#include <process.h>
#include <errno.h>
#include "fl2threading.h"
/* === Implementation === */
static unsigned __stdcall worker(void *arg)
{
ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg;
thread->arg = thread->start_routine(thread->arg);
return 0;
}
int FL2_pthread_create(ZSTD_pthread_t* thread, const void* unused,
void* (*start_routine) (void*), void* arg)
{
(void)unused;
thread->arg = arg;
thread->start_routine = start_routine;
thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL);
if (!thread->handle)
return errno;
else
return 0;
}
int FL2_pthread_join(ZSTD_pthread_t thread, void **value_ptr)
{
DWORD result;
if (!thread.handle) return 0;
result = WaitForSingleObject(thread.handle, INFINITE);
switch (result) {
case WAIT_OBJECT_0:
if (value_ptr) *value_ptr = thread.arg;
return 0;
case WAIT_ABANDONED:
return EINVAL;
default:
return GetLastError();
}
}
#endif /* FL2_SINGLETHREAD */

120
C/fast-lzma2/fl2threading.h Normal file
View File

@@ -0,0 +1,120 @@
/**
* Copyright (c) 2016 Tino Reichardt
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
*
* You can contact the author at:
* - zstdmt source repository: https://github.com/mcmilk/zstdmt
*/
#ifndef THREADING_H_938743
#define THREADING_H_938743
#if defined (__cplusplus)
extern "C" {
#endif
#if !defined(FL2_SINGLETHREAD) && defined(_WIN32)
/**
* Windows minimalist Pthread Wrapper, based on :
* http://www.cse.wustl.edu/~schmidt/win32-cv-1.html
*/
#ifdef WINVER
# undef WINVER
#endif
#define WINVER 0x0600
#ifdef _WIN32_WINNT
# undef _WIN32_WINNT
#endif
#define _WIN32_WINNT 0x0600
#ifndef WIN32_LEAN_AND_MEAN
# define WIN32_LEAN_AND_MEAN
#endif
#include <windows.h>
/* mutex */
#define ZSTD_pthread_mutex_t CRITICAL_SECTION
#define ZSTD_pthread_mutex_init(a, b) (InitializeCriticalSection((a)), 0)
#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a))
#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a))
#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a))
/* condition variable */
#define ZSTD_pthread_cond_t CONDITION_VARIABLE
#define ZSTD_pthread_cond_init(a, b) (InitializeConditionVariable((a)), 0)
#define ZSTD_pthread_cond_destroy(a) /* No delete */
#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE)
#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a))
#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a))
/* FL2_pthread_create() and FL2_pthread_join() */
typedef struct {
HANDLE handle;
void* (*start_routine)(void*);
void* arg;
} ZSTD_pthread_t;
int FL2_pthread_create(ZSTD_pthread_t* thread, const void* unused,
void* (*start_routine) (void*), void* arg);
int FL2_pthread_join(ZSTD_pthread_t thread, void** value_ptr);
/**
* add here more wrappers as required
*/
#elif !defined(FL2_SINGLETHREAD) /* posix assumed ; need a better detection method */
/* === POSIX Systems === */
# include <pthread.h>
#define ZSTD_pthread_mutex_t pthread_mutex_t
#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b))
#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a))
#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a))
#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a))
#define ZSTD_pthread_cond_t pthread_cond_t
#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b))
#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a))
#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b))
#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a))
#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a))
#define ZSTD_pthread_t pthread_t
#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d))
#define ZSTD_pthread_join(a, b) pthread_join((a),(b))
#else /* FL2_SINGLETHREAD defined */
/* No multithreading support */
typedef int ZSTD_pthread_mutex_t;
#define ZSTD_pthread_mutex_init(a, b) ((void)a, 0)
#define ZSTD_pthread_mutex_destroy(a)
#define ZSTD_pthread_mutex_lock(a)
#define ZSTD_pthread_mutex_unlock(a)
typedef int ZSTD_pthread_cond_t;
#define ZSTD_pthread_cond_init(a, b) ((void)a, 0)
#define ZSTD_pthread_cond_destroy(a)
#define ZSTD_pthread_cond_wait(a, b)
#define ZSTD_pthread_cond_signal(a)
#define ZSTD_pthread_cond_broadcast(a)
/* do not use ZSTD_pthread_t */
#endif /* FL2_SINGLETHREAD */
#if defined (__cplusplus)
}
#endif
#endif /* THREADING_H_938743 */

2047
C/fast-lzma2/lzma2_enc.c Normal file
View File

File diff suppressed because it is too large Load Diff

64
C/fast-lzma2/lzma2_enc.h Normal file
View File

@@ -0,0 +1,64 @@
/* lzma2_enc.h -- LZMA2 Encoder
Based on LzmaEnc.h and Lzma2Enc.h : Igor Pavlov
Modified for FL2 by Conor McCarthy
Public domain
*/
#ifndef RADYX_LZMA2_ENCODER_H
#define RADYX_LZMA2_ENCODER_H
#include "mem.h"
#include "data_block.h"
#include "radix_mf.h"
#if defined (__cplusplus)
extern "C" {
#endif
#define kFastDistBits 12U
#define LZMA2_END_MARKER '\0'
#define LZMA_MIN_DICT_BITS 12
typedef struct FL2_lzmaEncoderCtx_s FL2_lzmaEncoderCtx;
typedef enum {
FL2_fast,
FL2_opt,
FL2_ultra
} FL2_strategy;
typedef struct
{
unsigned lc;
unsigned lp;
unsigned pb;
unsigned fast_length;
unsigned match_cycles;
FL2_strategy strategy;
unsigned second_dict_bits;
unsigned random_filter;
} FL2_lzma2Parameters;
FL2_lzmaEncoderCtx* FL2_lzma2Create();
void FL2_lzma2Free(FL2_lzmaEncoderCtx* enc);
int FL2_lzma2HashAlloc(FL2_lzmaEncoderCtx* enc, const FL2_lzma2Parameters* options);
size_t FL2_lzma2Encode(FL2_lzmaEncoderCtx* enc,
FL2_matchTable* tbl,
const FL2_dataBlock block,
const FL2_lzma2Parameters* options,
FL2_progressFn progress, void* opaque, size_t base, U32 weight);
BYTE FL2_getDictSizeProp(size_t dictionary_size);
size_t FL2_lzma2MemoryUsage(unsigned chain_log, FL2_strategy strategy, unsigned thread_count);
#if defined (__cplusplus)
}
#endif
#endif /* RADYX_LZMA2_ENCODER_H */

362
C/fast-lzma2/mem.h Normal file
View File

@@ -0,0 +1,362 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef MEM_H_MODULE
#define MEM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include <stddef.h> /* size_t, ptrdiff_t */
#include <string.h> /* memcpy */
/*-****************************************
* Compiler specifics
******************************************/
#if defined(_MSC_VER) /* Visual Studio */
# include <stdlib.h> /* _byteswap_ulong */
# include <intrin.h> /* _byteswap_* */
#endif
#if defined(__GNUC__)
# define MEM_STATIC static __inline __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define MEM_STATIC static inline
#elif defined(_MSC_VER)
# define MEM_STATIC static __inline
#else
# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/* code only tested on 32 and 64 bits systems */
#define MEM_STATIC_ASSERT(c) { enum { MEM_static_assert = 1/(int)(!!(c)) }; }
MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); }
/*-**************************************************************
* Basic Types
*****************************************************************/
#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
# include <stdint.h>
typedef uint8_t BYTE;
typedef uint16_t U16;
typedef int16_t S16;
typedef uint32_t U32;
typedef int32_t S32;
typedef uint64_t U64;
typedef int64_t S64;
#else
typedef unsigned char BYTE;
typedef unsigned short U16;
typedef signed short S16;
typedef unsigned int U32;
typedef signed int S32;
typedef unsigned long long U64;
typedef signed long long S64;
#endif
/*-**************************************************************
* Memory I/O
*****************************************************************/
/* MEM_FORCE_MEMORY_ACCESS :
* By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable.
* Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal.
* The below switch allow to select different access method for improved performance.
* Method 0 (default) : use `memcpy()`. Safe and portable.
* Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable).
* This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`.
* Method 2 : direct access. This method is portable but violate C standard.
* It can generate buggy code on targets depending on alignment.
* In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6)
* See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details.
* Prefer these methods in priority order (0 > 1 > 2)
*/
#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) )
# define MEM_FORCE_MEMORY_ACCESS 2
# elif defined(__INTEL_COMPILER) || defined(__GNUC__)
# define MEM_FORCE_MEMORY_ACCESS 1
# endif
#endif
MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; }
MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; }
MEM_STATIC unsigned MEM_isLittleEndian(void)
{
const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */
return one.c[0];
}
#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2)
/* violates C standard, by lying on structure alignment.
Only use if no other choice to achieve best performance on target platform */
MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; }
MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; }
MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; }
MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; }
#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1)
/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */
/* currently only defined for gcc and icc */
#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32))
__pragma( pack(push, 1) )
typedef struct { U16 v; } unalign16;
typedef struct { U32 v; } unalign32;
typedef struct { U64 v; } unalign64;
typedef struct { size_t v; } unalignArch;
__pragma( pack(pop) )
#else
typedef struct { U16 v; } __attribute__((packed)) unalign16;
typedef struct { U32 v; } __attribute__((packed)) unalign32;
typedef struct { U64 v; } __attribute__((packed)) unalign64;
typedef struct { size_t v; } __attribute__((packed)) unalignArch;
#endif
MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; }
MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; }
MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; }
MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; }
MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; }
MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; }
MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; }
#else
/* default method, safe and standard.
can sometimes prove slower */
MEM_STATIC U16 MEM_read16(const void* memPtr)
{
U16 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U32 MEM_read32(const void* memPtr)
{
U32 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC U64 MEM_read64(const void* memPtr)
{
U64 val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC size_t MEM_readST(const void* memPtr)
{
size_t val; memcpy(&val, memPtr, sizeof(val)); return val;
}
MEM_STATIC void MEM_write16(void* memPtr, U16 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write32(void* memPtr, U32 value)
{
memcpy(memPtr, &value, sizeof(value));
}
MEM_STATIC void MEM_write64(void* memPtr, U64 value)
{
memcpy(memPtr, &value, sizeof(value));
}
#endif /* MEM_FORCE_MEMORY_ACCESS */
MEM_STATIC U32 MEM_swap32(U32 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_ulong(in);
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
return __builtin_bswap32(in);
#else
return ((in << 24) & 0xff000000 ) |
((in << 8) & 0x00ff0000 ) |
((in >> 8) & 0x0000ff00 ) |
((in >> 24) & 0x000000ff );
#endif
}
MEM_STATIC U64 MEM_swap64(U64 in)
{
#if defined(_MSC_VER) /* Visual Studio */
return _byteswap_uint64(in);
#elif defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)
return __builtin_bswap64(in);
#else
return ((in << 56) & 0xff00000000000000ULL) |
((in << 40) & 0x00ff000000000000ULL) |
((in << 24) & 0x0000ff0000000000ULL) |
((in << 8) & 0x000000ff00000000ULL) |
((in >> 8) & 0x00000000ff000000ULL) |
((in >> 24) & 0x0000000000ff0000ULL) |
((in >> 40) & 0x000000000000ff00ULL) |
((in >> 56) & 0x00000000000000ffULL);
#endif
}
MEM_STATIC size_t MEM_swapST(size_t in)
{
if (MEM_32bits())
return (size_t)MEM_swap32((U32)in);
else
return (size_t)MEM_swap64((U64)in);
}
/*=== Little endian r/w ===*/
MEM_STATIC U16 MEM_readLE16(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read16(memPtr);
else {
const BYTE* p = (const BYTE*)memPtr;
return (U16)(p[0] + (p[1]<<8));
}
}
MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val)
{
if (MEM_isLittleEndian()) {
MEM_write16(memPtr, val);
} else {
BYTE* p = (BYTE*)memPtr;
p[0] = (BYTE)val;
p[1] = (BYTE)(val>>8);
}
}
MEM_STATIC U32 MEM_readLE24(const void* memPtr)
{
return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16);
}
MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val)
{
MEM_writeLE16(memPtr, (U16)val);
((BYTE*)memPtr)[2] = (BYTE)(val>>16);
}
MEM_STATIC U32 MEM_readLE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read32(memPtr);
else
return MEM_swap32(MEM_read32(memPtr));
}
MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, val32);
else
MEM_write32(memPtr, MEM_swap32(val32));
}
MEM_STATIC U64 MEM_readLE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_read64(memPtr);
else
return MEM_swap64(MEM_read64(memPtr));
}
MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, val64);
else
MEM_write64(memPtr, MEM_swap64(val64));
}
MEM_STATIC size_t MEM_readLEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readLE32(memPtr);
else
return (size_t)MEM_readLE64(memPtr);
}
MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeLE32(memPtr, (U32)val);
else
MEM_writeLE64(memPtr, (U64)val);
}
/*=== Big endian r/w ===*/
MEM_STATIC U32 MEM_readBE32(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap32(MEM_read32(memPtr));
else
return MEM_read32(memPtr);
}
MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32)
{
if (MEM_isLittleEndian())
MEM_write32(memPtr, MEM_swap32(val32));
else
MEM_write32(memPtr, val32);
}
MEM_STATIC U64 MEM_readBE64(const void* memPtr)
{
if (MEM_isLittleEndian())
return MEM_swap64(MEM_read64(memPtr));
else
return MEM_read64(memPtr);
}
MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64)
{
if (MEM_isLittleEndian())
MEM_write64(memPtr, MEM_swap64(val64));
else
MEM_write64(memPtr, val64);
}
MEM_STATIC size_t MEM_readBEST(const void* memPtr)
{
if (MEM_32bits())
return (size_t)MEM_readBE32(memPtr);
else
return (size_t)MEM_readBE64(memPtr);
}
MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val)
{
if (MEM_32bits())
MEM_writeBE32(memPtr, (U32)val);
else
MEM_writeBE64(memPtr, (U64)val);
}
#if defined (__cplusplus)
}
#endif
#endif /* MEM_H_MODULE */

153
C/fast-lzma2/platform.h Normal file
View File

@@ -0,0 +1,153 @@
/*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef PLATFORM_H_MODULE
#define PLATFORM_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/* **************************************
* Compiler Options
****************************************/
#if defined(_MSC_VER)
# define _CRT_SECURE_NO_WARNINGS /* Disable Visual Studio warning messages for fopen, strncpy, strerror */
# define _CRT_SECURE_NO_DEPRECATE /* VS2005 - must be declared before <io.h> and <windows.h> */
# if (_MSC_VER <= 1800) /* (1800 = Visual Studio 2013) */
# define snprintf sprintf_s /* snprintf unsupported by Visual <= 2013 */
# endif
#endif
/* **************************************
* Detect 64-bit OS
* http://nadeausoftware.com/articles/2012/02/c_c_tip_how_detect_processor_type_using_compiler_predefined_macros
****************************************/
#if defined __ia64 || defined _M_IA64 /* Intel Itanium */ \
|| defined __powerpc64__ || defined __ppc64__ || defined __PPC64__ /* POWER 64-bit */ \
|| (defined __sparc && (defined __sparcv9 || defined __sparc_v9__ || defined __arch64__)) || defined __sparc64__ /* SPARC 64-bit */ \
|| defined __x86_64__s || defined _M_X64 /* x86 64-bit */ \
|| defined __arm64__ || defined __aarch64__ || defined __ARM64_ARCH_8__ /* ARM 64-bit */ \
|| (defined __mips && (__mips == 64 || __mips == 4 || __mips == 3)) /* MIPS 64-bit */ \
|| defined _LP64 || defined __LP64__ /* NetBSD, OpenBSD */ || defined __64BIT__ /* AIX */ || defined _ADDR64 /* Cray */ \
|| (defined __SIZEOF_POINTER__ && __SIZEOF_POINTER__ == 8) /* gcc */
# if !defined(__64BIT__)
# define __64BIT__ 1
# endif
#endif
/* *********************************************************
* Turn on Large Files support (>4GB) for 32-bit Linux/Unix
***********************************************************/
#if !defined(__64BIT__) || defined(__MINGW32__) /* No point defining Large file for 64 bit but MinGW-w64 requires it */
# if !defined(_FILE_OFFSET_BITS)
# define _FILE_OFFSET_BITS 64 /* turn off_t into a 64-bit type for ftello, fseeko */
# endif
# if !defined(_LARGEFILE_SOURCE) /* obsolete macro, replaced with _FILE_OFFSET_BITS */
# define _LARGEFILE_SOURCE 1 /* Large File Support extension (LFS) - fseeko, ftello */
# endif
# if defined(_AIX) || defined(__hpux)
# define _LARGE_FILES /* Large file support on 32-bits AIX and HP-UX */
# endif
#endif
/* ************************************************************
* Detect POSIX version
* PLATFORM_POSIX_VERSION = -1 for non-Unix e.g. Windows
* PLATFORM_POSIX_VERSION = 0 for Unix-like non-POSIX
* PLATFORM_POSIX_VERSION >= 1 is equal to found _POSIX_VERSION
***************************************************************/
#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \
|| defined(__midipix__) || defined(__VMS))
# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1–2001 (SUSv3) conformant */ \
|| defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */
# define PLATFORM_POSIX_VERSION 200112L
# else
# if defined(__linux__) || defined(__linux)
# ifndef _POSIX_C_SOURCE
# define _POSIX_C_SOURCE 200112L /* use feature test macro */
# endif
# endif
# include <unistd.h> /* declares _POSIX_VERSION */
# if defined(_POSIX_VERSION) /* POSIX compliant */
# define PLATFORM_POSIX_VERSION _POSIX_VERSION
# else
# define PLATFORM_POSIX_VERSION 0
# endif
# endif
#endif
#if !defined(PLATFORM_POSIX_VERSION)
# define PLATFORM_POSIX_VERSION -1
#endif
/*-*********************************************
* Detect if isatty() and fileno() are available
************************************************/
#if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 1)) || (PLATFORM_POSIX_VERSION >= 200112L) || defined(__DJGPP__)
# include <unistd.h> /* isatty */
# define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
#elif defined(MSDOS) || defined(OS2) || defined(__CYGWIN__)
# include <io.h> /* _isatty */
# define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
#elif defined(WIN32) || defined(_WIN32)
# include <io.h> /* _isatty */
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <stdio.h> /* FILE */
static __inline int IS_CONSOLE(FILE* stdStream)
{
DWORD dummy;
return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy);
}
#else
# define IS_CONSOLE(stdStream) 0
#endif
/******************************
* OS-specific Includes
******************************/
#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32)
# include <fcntl.h> /* _O_BINARY */
# include <io.h> /* _setmode, _fileno, _get_osfhandle */
# if !defined(__DJGPP__)
# include <windows.h> /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */
# include <winioctl.h> /* FSCTL_SET_SPARSE */
# define SET_BINARY_MODE(file) { int unused=_setmode(_fileno(file), _O_BINARY); (void)unused; }
# define SET_SPARSE_FILE_MODE(file) { DWORD dw; DeviceIoControl((HANDLE) _get_osfhandle(_fileno(file)), FSCTL_SET_SPARSE, 0, 0, 0, 0, &dw, 0); }
# else
# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY)
# define SET_SPARSE_FILE_MODE(file)
# endif
#else
# define SET_BINARY_MODE(file)
# define SET_SPARSE_FILE_MODE(file)
#endif
#ifndef ZSTD_SPARSE_DEFAULT
# if (defined(__APPLE__) && defined(__MACH__))
# define ZSTD_SPARSE_DEFAULT 0
# else
# define ZSTD_SPARSE_DEFAULT 1
# endif
#endif
#if defined (__cplusplus)
}
#endif
#endif /* PLATFORM_H_MODULE */

61
C/fast-lzma2/radix_bitpack.c Normal file
View File

@@ -0,0 +1,61 @@
/*
* Copyright (c) 2018, Conor McCarthy
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "mem.h" /* U32, U64 */
#include "fl2threading.h"
#include "fl2_internal.h"
#include "radix_internal.h"
typedef struct FL2_matchTable_s FL2_matchTable;
#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define RMF_BITPACK
#define RADIX_MAX_LENGTH BITPACK_MAX_LENGTH
#define InitMatchLink(index, link) tbl->table[index] = link
#define GetMatchLink(link) (tbl->table[link] & RADIX_LINK_MASK)
#define GetInitialMatchLink(index) tbl->table[index]
#define GetMatchLength(index) (tbl->table[index] >> RADIX_LINK_BITS)
#define SetMatchLink(index, link, length) tbl->table[index] = (link) | ((U32)(length) << RADIX_LINK_BITS)
#define SetMatchLength(index, link, length) tbl->table[index] = (link) | ((U32)(length) << RADIX_LINK_BITS)
#define SetMatchLinkAndLength(index, link, length) tbl->table[index] = (link) | ((U32)(length) << RADIX_LINK_BITS)
#define SetNull(index) tbl->table[index] = RADIX_NULL_LINK
#define IsNull(index) (tbl->table[index] == RADIX_NULL_LINK)
BYTE* RMF_bitpackAsOutputBuffer(FL2_matchTable* const tbl, size_t const index)
{
return (BYTE*)(tbl->table + index);
}
/* Restrict the match lengths so that they don't reach beyond index */
void RMF_bitpackLimitLengths(FL2_matchTable* const tbl, size_t const index)
{
DEBUGLOG(5, "RMF_limitLengths : end %u, max length %u", (U32)index, RADIX_MAX_LENGTH);
SetNull(index - 1);
for (U32 length = 2; length < RADIX_MAX_LENGTH && length <= index; ++length) {
U32 const link = tbl->table[index - length];
if (link != RADIX_NULL_LINK) {
tbl->table[index - length] = (MIN(length, link >> RADIX_LINK_BITS) << RADIX_LINK_BITS) | (link & RADIX_LINK_MASK);
}
}
}
#include "radix_engine.h"

1075
C/fast-lzma2/radix_engine.h Normal file
View File

File diff suppressed because it is too large Load Diff

143
C/fast-lzma2/radix_internal.h Normal file
View File

@@ -0,0 +1,143 @@
/*
* Copyright (c) 2018, Conor McCarthy
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef RADIX_INTERNAL_H
#define RADIX_INTERNAL_H
#include "atomic.h"
#include "radix_mf.h"
#if defined (__cplusplus)
extern "C" {
#endif
#define DICTIONARY_LOG_MIN 12U
#define DICTIONARY_LOG_MAX_64 30U
#define DICTIONARY_LOG_MAX_32 27U
#define DEFAULT_BUFFER_LOG 8U
#define DEFAULT_BLOCK_OVERLAP 2U
#define DEFAULT_SEARCH_DEPTH 32U
#define DEFAULT_DIVIDEANDCONQUER 1
#define MAX_REPEAT 32
#define RADIX16_TABLE_SIZE (1UL << 16)
#define RADIX8_TABLE_SIZE (1UL << 8)
#define STACK_SIZE (RADIX16_TABLE_SIZE * 3)
#define MAX_BRUTE_FORCE_LIST_SIZE 5
#define BUFFER_LINK_MASK 0xFFFFFFU
#define MATCH_BUFFER_OVERLAP 6
#define BITPACK_MAX_LENGTH 63UL
#define STRUCTURED_MAX_LENGTH 255UL
#define RADIX_LINK_BITS 26
#define RADIX_LINK_MASK ((1UL << RADIX_LINK_BITS) - 1)
#define RADIX_NULL_LINK 0xFFFFFFFFUL
#define UNIT_BITS 2
#define UNIT_MASK ((1UL << UNIT_BITS) - 1)
typedef struct
{
U32 head;
U32 count;
} RMF_tableHead;
union src_data_u {
BYTE chars[4];
U32 u32;
};
typedef struct
{
U32 from;
union src_data_u src;
U32 next;
} RMF_buildMatch;
typedef struct
{
U32 prev_index;
U32 list_count;
} RMF_listTail;
typedef struct
{
U32 links[1 << UNIT_BITS];
BYTE lengths[1 << UNIT_BITS];
} RMF_unit;
typedef struct
{
unsigned max_len;
U32* table;
size_t match_buffer_size;
size_t match_buffer_limit;
RMF_listTail tails_8[RADIX8_TABLE_SIZE];
RMF_tableHead stack[STACK_SIZE];
RMF_listTail tails_16[RADIX16_TABLE_SIZE];
RMF_buildMatch match_buffer[1];
} RMF_builder;
struct FL2_matchTable_s
{
FL2_atomic st_index;
long end_index;
int isStruct;
int allocStruct;
unsigned thread_count;
RMF_parameters params;
RMF_builder** builders;
U32 stack[RADIX16_TABLE_SIZE];
RMF_tableHead list_heads[RADIX16_TABLE_SIZE];
U32 table[1];
};
size_t RMF_bitpackInit(struct FL2_matchTable_s* const tbl, const void* data, size_t const start, size_t const end);
size_t RMF_structuredInit(struct FL2_matchTable_s* const tbl, const void* data, size_t const start, size_t const end);
int RMF_bitpackBuildTable(struct FL2_matchTable_s* const tbl,
size_t const job,
unsigned const multi_thread,
FL2_dataBlock const block,
FL2_progressFn progress, void* opaque, U32 weight, size_t init_done);
int RMF_structuredBuildTable(struct FL2_matchTable_s* const tbl,
size_t const job,
unsigned const multi_thread,
FL2_dataBlock const block,
FL2_progressFn progress, void* opaque, U32 weight, size_t init_done);
void RMF_recurseListChunk(RMF_builder* const tbl,
const BYTE* const data_block,
size_t const block_start,
BYTE const depth,
BYTE const max_depth,
U32 const list_count,
size_t const stack_base);
int RMF_bitpackIntegrityCheck(const struct FL2_matchTable_s* const tbl, const BYTE* const data, size_t index, size_t const end, unsigned const max_depth);
int RMF_structuredIntegrityCheck(const struct FL2_matchTable_s* const tbl, const BYTE* const data, size_t index, size_t const end, unsigned const max_depth);
void RMF_bitpackLimitLengths(struct FL2_matchTable_s* const tbl, size_t const index);
void RMF_structuredLimitLengths(struct FL2_matchTable_s* const tbl, size_t const index);
BYTE* RMF_bitpackAsOutputBuffer(struct FL2_matchTable_s* const tbl, size_t const index);
BYTE* RMF_structuredAsOutputBuffer(struct FL2_matchTable_s* const tbl, size_t const index);
size_t RMF_bitpackGetMatch(const struct FL2_matchTable_s* const tbl,
const BYTE* const data,
size_t const index,
size_t const limit,
unsigned const max_depth,
size_t* const offset_ptr);
size_t RMF_structuredGetMatch(const struct FL2_matchTable_s* const tbl,
const BYTE* const data,
size_t const index,
size_t const limit,
unsigned const max_depth,
size_t* const offset_ptr);
#if defined (__cplusplus)
}
#endif
#endif /* RADIX_INTERNAL_H */

672
C/fast-lzma2/radix_mf.c Normal file
View File

@@ -0,0 +1,672 @@
/*
* Copyright (c) 2018, Conor McCarthy
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdlib.h> /* malloc, free */
#include "fast-lzma2.h"
#include "mem.h" /* U32, U64, MEM_64bits */
#include "fl2_internal.h"
#include "radix_internal.h"
#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" /* warning: 'rpt_head_next' may be used uninitialized in this function */
#elif defined(_MSC_VER)
# pragma warning(disable : 4701) /* disable: C4701: potentially uninitialized local variable */
#endif
#define MIN_MATCH_BUFFER_SIZE 256U /* min buffer size at least FL2_SEARCH_DEPTH_MAX + 2 for bounded build */
#define MAX_MATCH_BUFFER_SIZE (1UL << 24) /* max buffer size constrained by 24-bit link values */
#define REPEAT_CHECK_TABLE ((1 << 1) | (1 << 2) | (1 << 4) | (1 << 8) | (1 << 16) | (1ULL << 32))
static void RMF_initTailTable(RMF_builder* const tbl)
{
for (size_t i = 0; i < RADIX8_TABLE_SIZE; i += 2) {
tbl->tails_8[i].prev_index = RADIX_NULL_LINK;
tbl->tails_8[i + 1].prev_index = RADIX_NULL_LINK;
}
for (size_t i = 0; i < RADIX16_TABLE_SIZE; i += 2) {
tbl->tails_16[i].prev_index = RADIX_NULL_LINK;
tbl->tails_16[i + 1].prev_index = RADIX_NULL_LINK;
}
}
static RMF_builder* RMF_createBuilder(size_t match_buffer_size)
{
match_buffer_size = MIN(match_buffer_size, MAX_MATCH_BUFFER_SIZE);
match_buffer_size = MAX(match_buffer_size, MIN_MATCH_BUFFER_SIZE);
{ RMF_builder* const builder = (RMF_builder*)malloc(
sizeof(RMF_builder) + (match_buffer_size - 1) * sizeof(RMF_buildMatch));
builder->match_buffer_size = match_buffer_size;
builder->match_buffer_limit = match_buffer_size;
RMF_initTailTable(builder);
return builder;
}
}
static void RMF_freeBuilderTable(RMF_builder** const builders, unsigned const size)
{
if (builders == NULL)
return;
for (unsigned i = 0; i < size; ++i) {
free(builders[i]);
}
free(builders);
}
static RMF_builder** RMF_createBuilderTable(U32* const matchTable, size_t const match_buffer_size, unsigned const max_len, unsigned const size)
{
RMF_builder** builders = (RMF_builder**)malloc(size * sizeof(RMF_builder*));
DEBUGLOG(3, "RMF_createBuilderTable : match_buffer_size %u, builders %u", (U32)match_buffer_size, size);
if (builders == NULL)
return NULL;
for (unsigned i = 0; i < size; ++i)
builders[i] = NULL;
for (unsigned i = 0; i < size; ++i) {
builders[i] = RMF_createBuilder(match_buffer_size);
if (builders[i] == NULL) {
RMF_freeBuilderTable(builders, i);
return NULL;
}
builders[i]->table = matchTable;
builders[i]->max_len = max_len;
}
return builders;
}
static int RMF_isStruct(unsigned dictionary_log, unsigned depth)
{
return dictionary_log > RADIX_LINK_BITS || depth > BITPACK_MAX_LENGTH;
}
static int RMF_isStructParam(const RMF_parameters* const params)
{
return RMF_isStruct(params->dictionary_log, params->depth);
}
/** RMF_clampCParams() :
* make CParam values within valid range.
* @return : valid CParams */
static RMF_parameters RMF_clampParams(RMF_parameters params)
{
# define CLAMP(val,min,max) { \
if (val<(min)) val=(min); \
else if (val>(max)) val=(max); \
}
CLAMP(params.dictionary_log, DICTIONARY_LOG_MIN, MEM_64bits() ? DICTIONARY_LOG_MAX_64 : DICTIONARY_LOG_MAX_32);
CLAMP(params.match_buffer_log, FL2_BUFFER_SIZE_LOG_MIN, FL2_BUFFER_SIZE_LOG_MAX);
CLAMP(params.overlap_fraction, FL2_BLOCK_OVERLAP_MIN, FL2_BLOCK_OVERLAP_MAX);
CLAMP(params.depth, FL2_SEARCH_DEPTH_MIN, FL2_SEARCH_DEPTH_MAX);
return params;
}
static size_t RMF_applyParameters_internal(FL2_matchTable* const tbl, const RMF_parameters* const params)
{
int const isStruct = RMF_isStructParam(params);
unsigned const dictionary_log = tbl->params.dictionary_log;
/* dictionary is allocated with the struct and is immutable */
if (params->dictionary_log > tbl->params.dictionary_log
|| (params->dictionary_log == tbl->params.dictionary_log && isStruct > tbl->allocStruct))
return FL2_ERROR(parameter_unsupported);
{ size_t const match_buffer_size = (size_t)1 << (params->dictionary_log - params->match_buffer_log);
tbl->params = *params;
tbl->params.dictionary_log = dictionary_log;
tbl->isStruct = isStruct;
if (tbl->builders == NULL
|| match_buffer_size > tbl->builders[0]->match_buffer_size)
{
RMF_freeBuilderTable(tbl->builders, tbl->thread_count);
tbl->builders = RMF_createBuilderTable(tbl->table, match_buffer_size, tbl->isStruct ? STRUCTURED_MAX_LENGTH : BITPACK_MAX_LENGTH, tbl->thread_count);
if (tbl->builders == NULL) {
return FL2_ERROR(memory_allocation);
}
}
else {
for (unsigned i = 0; i < tbl->thread_count; ++i) {
tbl->builders[i]->match_buffer_limit = match_buffer_size;
tbl->builders[i]->max_len = tbl->isStruct ? STRUCTURED_MAX_LENGTH : BITPACK_MAX_LENGTH;
}
}
}
return 0;
}
static void RMF_reduceDict(RMF_parameters* const params, size_t const dict_reduce)
{
if (dict_reduce)
while (params->dictionary_log > DICTIONARY_LOG_MIN && (size_t)1 << (params->dictionary_log - 1) >= dict_reduce) {
--params->dictionary_log;
params->match_buffer_log = MAX(params->match_buffer_log - 1, FL2_BUFFER_SIZE_LOG_MIN);
}
}
FL2_matchTable* RMF_createMatchTable(const RMF_parameters* const p, size_t const dict_reduce, unsigned const thread_count)
{
int isStruct;
size_t dictionary_size;
size_t table_bytes;
FL2_matchTable* tbl;
RMF_parameters params = RMF_clampParams(*p);
RMF_reduceDict(&params, dict_reduce);
isStruct = RMF_isStructParam(&params);
dictionary_size = (size_t)1 << params.dictionary_log;
DEBUGLOG(3, "RMF_createMatchTable : isStruct %d, dict %u", isStruct, (U32)dictionary_size);
table_bytes = isStruct ? ((dictionary_size + 3U) / 4U) * sizeof(RMF_unit)
: dictionary_size * sizeof(U32);
tbl = (FL2_matchTable*)malloc(
sizeof(FL2_matchTable) + table_bytes - sizeof(U32));
if (!tbl) return NULL;
tbl->isStruct = isStruct;
tbl->allocStruct = isStruct;
tbl->thread_count = thread_count + !thread_count;
tbl->params = params;
tbl->builders = NULL;
RMF_applyParameters_internal(tbl, &params);
for (size_t i = 0; i < RADIX16_TABLE_SIZE; i += 2) {
tbl->list_heads[i].head = RADIX_NULL_LINK;
tbl->list_heads[i].count = 0;
tbl->list_heads[i + 1].head = RADIX_NULL_LINK;
tbl->list_heads[i + 1].count = 0;
}
return tbl;
}
void RMF_freeMatchTable(FL2_matchTable* const tbl)
{
if (tbl == NULL)
return;
DEBUGLOG(3, "RMF_freeMatchTable");
RMF_freeBuilderTable(tbl->builders, tbl->thread_count);
free(tbl);
}
BYTE RMF_compatibleParameters(const FL2_matchTable* const tbl, const RMF_parameters * const p, size_t const dict_reduce)
{
RMF_parameters params = RMF_clampParams(*p);
RMF_reduceDict(&params, dict_reduce);
return tbl->params.dictionary_log > params.dictionary_log
|| (tbl->params.dictionary_log == params.dictionary_log && tbl->allocStruct >= RMF_isStructParam(&params));
}
size_t RMF_applyParameters(FL2_matchTable* const tbl, const RMF_parameters* const p, size_t const dict_reduce)
{
RMF_parameters params = RMF_clampParams(*p);
RMF_reduceDict(&params, dict_reduce);
return RMF_applyParameters_internal(tbl, &params);
}
size_t RMF_threadCount(const FL2_matchTable* const tbl)
{
return tbl->thread_count;
}
size_t RMF_initTable(FL2_matchTable* const tbl, const void* const data, size_t const start, size_t const end)
{
DEBUGLOG(5, "RMF_initTable : start %u, size %u", (U32)start, (U32)end);
if (tbl->isStruct) {
return RMF_structuredInit(tbl, data, start, end);
}
else {
return RMF_bitpackInit(tbl, data, start, end);
}
}
static void HandleRepeat(RMF_buildMatch* const match_buffer,
const BYTE* const data_block,
size_t const next,
U32 count,
U32 const rpt_len,
U32 const depth,
U32 const max_len)
{
size_t index = next;
size_t next_i;
U32 length = depth + rpt_len;
const BYTE* const data = data_block + match_buffer[index].from;
const BYTE* const data_2 = data - rpt_len;
while (data[length] == data_2[length] && length < max_len)
++length;
for (; length <= max_len && count; --count) {
next_i = match_buffer[index].next & 0xFFFFFF;
match_buffer[index].next = (U32)next_i | (length << 24);
length += rpt_len;
index = next_i;
}
for (; count; --count) {
next_i = match_buffer[index].next & 0xFFFFFF;
match_buffer[index].next = (U32)next_i | (max_len << 24);
index = next_i;
}
}
typedef struct
{
size_t index;
const BYTE* data_src;
union src_data_u src;
} BruteForceMatch;
static void BruteForceBuffered(RMF_builder* const tbl,
const BYTE* const data_block,
size_t const block_start,
size_t index,
size_t list_count,
size_t const slot,
size_t const depth,
size_t const max_depth)
{
BruteForceMatch buffer[MAX_BRUTE_FORCE_LIST_SIZE + 1];
const BYTE* data_src = data_block + depth;
size_t limit = max_depth - depth;
const BYTE* start = data_src + block_start;
size_t i = 0;
for (;;) {
buffer[i].index = index;
buffer[i].data_src = data_src + tbl->match_buffer[index].from;
buffer[i].src.u32 = tbl->match_buffer[index].src.u32;
if (++i >= list_count) {
break;
}
index = tbl->match_buffer[index].next & 0xFFFFFF;
}
i = 0;
do {
size_t longest = 0;
size_t j = i + 1;
size_t longest_index = j;
const BYTE* data = buffer[i].data_src;
do {
size_t len_test = slot;
while (len_test < 4 && buffer[i].src.chars[len_test] == buffer[j].src.chars[len_test] && len_test - slot < limit) {
++len_test;
}
len_test -= slot;
if (len_test) {
const BYTE* data_2 = buffer[j].data_src;
while (data[len_test] == data_2[len_test] && len_test < limit) {
++len_test;
}
}
if (len_test > longest) {
longest_index = j;
longest = len_test;
if (len_test >= limit) {
break;
}
}
} while (++j < list_count);
if (longest > 0) {
index = buffer[i].index;
tbl->match_buffer[index].next = (U32)(buffer[longest_index].index | ((depth + longest) << 24));
}
++i;
} while (i < list_count - 1 && buffer[i].data_src >= start);
}
FORCE_INLINE_TEMPLATE
void RMF_recurseListChunk_generic(RMF_builder* const tbl,
const BYTE* const data_block,
size_t const block_start,
BYTE depth,
BYTE const max_depth,
U32 list_count,
size_t const stack_base)
{
/* Create an offset data buffer pointer for reading the next bytes */
const BYTE base_depth = depth;
size_t st_index = stack_base;
size_t index = 0;
++depth;
/* The last element is done separately and won't be copied back at the end */
--list_count;
do {
size_t const radix_8 = tbl->match_buffer[index].src.chars[0];
/* Seen this char before? */
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
++tbl->tails_8[radix_8].list_count;
/* Link the previous occurrence to this one and record the new length */
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
else {
tbl->tails_8[radix_8].list_count = 1;
/* Add the new sub list to the stack */
tbl->stack[st_index].head = (U32)index;
/* This will be converted to a count at the end */
tbl->stack[st_index].count = (U32)radix_8;
++st_index;
}
tbl->tails_8[radix_8].prev_index = (U32)index;
++index;
} while (index < list_count);
{ /* Do the last element */
size_t const radix_8 = tbl->match_buffer[index].src.chars[0];
/* Nothing to do if there was no previous */
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
}
/* Convert radix values on the stack to counts and reset any used tail slots */
for (size_t j = stack_base; j < st_index; ++j) {
tbl->tails_8[tbl->stack[j].count].prev_index = RADIX_NULL_LINK;
tbl->stack[j].count = (U32)tbl->tails_8[tbl->stack[j].count].list_count;
}
while (st_index > stack_base) {
const BYTE* data_src;
size_t link;
size_t slot;
U32 test;
/* Pop an item off the stack */
--st_index;
list_count = tbl->stack[st_index].count;
if (list_count < 2) {
/* Nothing to match with */
continue;
}
index = tbl->stack[st_index].head;
link = tbl->match_buffer[index].from;
if (link < block_start) {
/* Chain starts in the overlap region which is already encoded */
continue;
}
/* Check stack space. The first comparison is unnecessary but it's a constant so should be faster */
if (st_index > STACK_SIZE - RADIX8_TABLE_SIZE
&& st_index > STACK_SIZE - list_count)
{
/* Stack may not be able to fit all possible new items. This is very rare. */
continue;
}
depth = tbl->match_buffer[index].next >> 24;
slot = (depth - base_depth) & 3;
if (list_count <= MAX_BRUTE_FORCE_LIST_SIZE) {
/* Quicker to use brute force, each string compared with all previous strings */
BruteForceBuffered(tbl,
data_block,
block_start,
index,
list_count,
slot,
depth,
max_depth);
continue;
}
/* check for repeats at depth 4,8,16,32 etc */
test = max_depth != 6 && ((depth & 3) == 0) && ((REPEAT_CHECK_TABLE >> ((depth >> 2) & 31)) & 1) && (max_depth >= depth + (depth >> 1));
++depth;
/* Update the offset data buffer pointer */
data_src = data_block + depth;
/* Last pass is done separately */
if (!test && depth < max_depth) {
size_t const prev_st_index = st_index;
/* Last element done separately */
--list_count;
/* slot is the char cache index. If 3 then chars need to be loaded. */
if (slot == 3 && max_depth != 6) do {
size_t const radix_8 = tbl->match_buffer[index].src.chars[3];
size_t const next_index = tbl->match_buffer[index].next & BUFFER_LINK_MASK;
/* Pre-load the next link and data bytes to avoid waiting for RAM access */
tbl->match_buffer[index].src.u32 = MEM_read32(data_src + link);
size_t const next_link = tbl->match_buffer[next_index].from;
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev!=RADIX_NULL_LINK) {
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
else {
tbl->tails_8[radix_8].list_count = 1;
tbl->stack[st_index].head = (U32)index;
tbl->stack[st_index].count = (U32)radix_8;
++st_index;
}
tbl->tails_8[radix_8].prev_index = (U32)index;
index = next_index;
link = next_link;
} while (--list_count != 0);
else do {
size_t const radix_8 = tbl->match_buffer[index].src.chars[slot];
size_t const next_index = tbl->match_buffer[index].next & BUFFER_LINK_MASK;
/* Pre-load the next link to avoid waiting for RAM access */
size_t const next_link = tbl->match_buffer[next_index].from;
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
else {
tbl->tails_8[radix_8].list_count = 1;
tbl->stack[st_index].head = (U32)index;
tbl->stack[st_index].count = (U32)radix_8;
++st_index;
}
tbl->tails_8[radix_8].prev_index = (U32)index;
index = next_index;
link = next_link;
} while (--list_count != 0);
{ size_t const radix_8 = tbl->match_buffer[index].src.chars[slot];
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
if (slot == 3) {
tbl->match_buffer[index].src.u32 = MEM_read32(data_src + link);
}
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
}
for (size_t j = prev_st_index; j < st_index; ++j) {
tbl->tails_8[tbl->stack[j].count].prev_index = RADIX_NULL_LINK;
tbl->stack[j].count = (U32)tbl->tails_8[tbl->stack[j].count].list_count;
}
}
else if (test) {
S32 rpt = -1;
size_t rpt_head_next;
U32 rpt_dist = 0;
size_t const prev_st_index = st_index;
U32 const rpt_depth = depth - 1;
/* Last element done separately */
--list_count;
do {
size_t const radix_8 = tbl->match_buffer[index].src.chars[slot];
size_t const next_index = tbl->match_buffer[index].next & BUFFER_LINK_MASK;
size_t const next_link = tbl->match_buffer[next_index].from;
if ((link - next_link) > rpt_depth) {
if (rpt > 0) {
HandleRepeat(tbl->match_buffer, data_block, rpt_head_next, rpt, rpt_dist, rpt_depth, tbl->max_len);
}
rpt = -1;
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
else {
tbl->tails_8[radix_8].list_count = 1;
tbl->stack[st_index].head = (U32)index;
tbl->stack[st_index].count = (U32)radix_8;
++st_index;
}
tbl->tails_8[radix_8].prev_index = (U32)index;
index = next_index;
link = next_link;
}
else {
U32 const dist = (U32)(link - next_link);
if (rpt < 0 || dist != rpt_dist) {
if (rpt > 0) {
HandleRepeat(tbl->match_buffer, data_block, rpt_head_next, rpt, rpt_dist, rpt_depth, tbl->max_len);
}
rpt = 0;
rpt_head_next = next_index;
rpt_dist = dist;
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
else {
tbl->tails_8[radix_8].list_count = 1;
tbl->stack[st_index].head = (U32)index;
tbl->stack[st_index].count = (U32)radix_8;
++st_index;
}
tbl->tails_8[radix_8].prev_index = (U32)index;
}
else {
++rpt;
}
index = next_index;
link = next_link;
}
} while (--list_count != 0);
if (rpt > 0) {
HandleRepeat(tbl->match_buffer, data_block, rpt_head_next, rpt, rpt_dist, rpt_depth, tbl->max_len);
}
{ size_t const radix_8 = tbl->match_buffer[index].src.chars[slot];
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
if (slot == 3) {
tbl->match_buffer[index].src.u32 = MEM_read32(data_src + link);
}
++tbl->tails_8[radix_8].list_count;
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
}
for (size_t j = prev_st_index; j < st_index; ++j) {
tbl->tails_8[tbl->stack[j].count].prev_index = RADIX_NULL_LINK;
tbl->stack[j].count = (U32)tbl->tails_8[tbl->stack[j].count].list_count;
}
}
else {
size_t prev_st_index = st_index;
/* The last pass at max_depth */
do {
size_t const radix_8 = tbl->match_buffer[index].src.chars[slot];
size_t const next_index = tbl->match_buffer[index].next & BUFFER_LINK_MASK;
/* Pre-load the next link. */
/* The last element in tbl->match_buffer is circular so this is never an access violation. */
size_t const next_link = tbl->match_buffer[next_index].from;
U32 const prev = tbl->tails_8[radix_8].prev_index;
if (prev != RADIX_NULL_LINK) {
tbl->match_buffer[prev].next = (U32)index | ((U32)depth << 24);
}
else {
tbl->stack[st_index].count = (U32)radix_8;
++st_index;
}
tbl->tails_8[radix_8].prev_index = (U32)index;
index = next_index;
link = next_link;
} while (--list_count != 0);
for (size_t j = prev_st_index; j < st_index; ++j) {
tbl->tails_8[tbl->stack[j].count].prev_index = RADIX_NULL_LINK;
}
st_index = prev_st_index;
}
}
}
void RMF_recurseListChunk(RMF_builder* const tbl,
const BYTE* const data_block,
size_t const block_start,
BYTE const depth,
BYTE const max_depth,
U32 const list_count,
size_t const stack_base)
{
if (max_depth > 6) {
RMF_recurseListChunk_generic(tbl, data_block, block_start, depth, max_depth, list_count, stack_base);
}
else {
RMF_recurseListChunk_generic(tbl, data_block, block_start, depth, 6, list_count, stack_base);
}
}
/* Iterate the head table concurrently with other threads, and recurse each list until max_depth is reached */
int RMF_buildTable(FL2_matchTable* const tbl,
size_t const job,
unsigned const multi_thread,
FL2_dataBlock const block,
FL2_progressFn progress, void* opaque, U32 weight, size_t init_done)
{
DEBUGLOG(5, "RMF_buildTable : thread %u", (U32)job);
if (tbl->isStruct) {
return RMF_structuredBuildTable(tbl, job, multi_thread, block, progress, opaque, weight, init_done);
}
else {
return RMF_bitpackBuildTable(tbl, job, multi_thread, block, progress, opaque, weight, init_done);
}
}
int RMF_integrityCheck(const FL2_matchTable* const tbl, const BYTE* const data, size_t const index, size_t const end, unsigned const max_depth)
{
if (tbl->isStruct) {
return RMF_structuredIntegrityCheck(tbl, data, index, end, max_depth);
}
else {
return RMF_bitpackIntegrityCheck(tbl, data, index, end, max_depth);
}
}
size_t RMF_getMatch(FL2_matchTable* const tbl,
const BYTE* const data,
size_t const index,
size_t const limit,
unsigned max_depth,
size_t* const offset_ptr)
{
if (tbl->isStruct) {
return RMF_structuredGetMatch(tbl, data, index, limit, max_depth, offset_ptr);
}
else {
return RMF_bitpackGetMatch(tbl, data, index, limit, max_depth, offset_ptr);
}
}
void RMF_limitLengths(FL2_matchTable* const tbl, size_t const index)
{
if (tbl->isStruct) {
RMF_structuredLimitLengths(tbl, index);
}
else {
RMF_bitpackLimitLengths(tbl, index);
}
}
BYTE* RMF_getTableAsOutputBuffer(FL2_matchTable* const tbl, size_t const index)
{
if (tbl->isStruct) {
return RMF_structuredAsOutputBuffer(tbl, index);
}
else {
return RMF_bitpackAsOutputBuffer(tbl, index);
}
}
size_t RMF_memoryUsage(unsigned const dict_log, unsigned const buffer_log, unsigned const depth, unsigned thread_count)
{
size_t size = (size_t)(4U + RMF_isStruct(dict_log, depth)) << dict_log;
U32 buf_size = (U32)1 << (dict_log - buffer_log);
size += ((buf_size - 1) * sizeof(RMF_buildMatch) + sizeof(RMF_builder)) * thread_count;
return size;
}

60
C/fast-lzma2/radix_mf.h Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright (c) 2018, Conor McCarthy
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef RADIX_MF_H
#define RADIX_MF_H
#include "fast-lzma2.h"
#include "data_block.h"
#if defined (__cplusplus)
extern "C" {
#endif
typedef struct FL2_matchTable_s FL2_matchTable;
#define OVERLAP_FROM_DICT_LOG(d, o) (((size_t)1 << ((d) - 4)) * (o))
#define RMF_MIN_BYTES_PER_THREAD 1024
typedef struct
{
unsigned dictionary_log;
unsigned match_buffer_log;
unsigned overlap_fraction;
unsigned block_size_log;
unsigned divide_and_conquer;
unsigned depth;
#ifdef RMF_REFERENCE
unsigned use_ref_mf;
#endif
} RMF_parameters;
FL2_matchTable* RMF_createMatchTable(const RMF_parameters* const params, size_t const dict_reduce, unsigned const thread_count);
void RMF_freeMatchTable(FL2_matchTable* const tbl);
BYTE RMF_compatibleParameters(const FL2_matchTable* const tbl, const RMF_parameters* const params, size_t const dict_reduce);
size_t RMF_applyParameters(FL2_matchTable* const tbl, const RMF_parameters* const params, size_t const dict_reduce);
size_t RMF_threadCount(const FL2_matchTable * const tbl);
size_t RMF_initTable(FL2_matchTable* const tbl, const void* const data, size_t const start, size_t const end);
int RMF_buildTable(FL2_matchTable* const tbl,
size_t const job,
unsigned const multi_thread,
FL2_dataBlock const block,
FL2_progressFn progress, void* opaque, U32 weight, size_t init_done);
int RMF_integrityCheck(const FL2_matchTable* const tbl, const BYTE* const data, size_t const index, size_t const end, unsigned const max_depth);
void RMF_limitLengths(FL2_matchTable* const tbl, size_t const index);
BYTE* RMF_getTableAsOutputBuffer(FL2_matchTable* const tbl, size_t const index);
size_t RMF_memoryUsage(unsigned const dict_log, unsigned const buffer_log, unsigned const depth, unsigned thread_count);
#if defined (__cplusplus)
}
#endif
#endif /* RADIX_MF_H */

62
C/fast-lzma2/radix_struct.c Normal file
View File

@@ -0,0 +1,62 @@
/*
* Copyright (c) 2018, Conor McCarthy
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#include "mem.h" /* U32, U64 */
#include "fl2threading.h"
#include "fl2_internal.h"
#include "radix_internal.h"
typedef struct FL2_matchTable_s FL2_matchTable;
#undef MIN
#define MIN(a,b) ((a) < (b) ? (a) : (b))
#define RMF_STRUCTURED
#define RADIX_MAX_LENGTH STRUCTURED_MAX_LENGTH
#define InitMatchLink(index, link) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].links[(index) & UNIT_MASK] = (U32)(link)
#define GetMatchLink(index) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].links[(index) & UNIT_MASK]
#define GetInitialMatchLink(index) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].links[(index) & UNIT_MASK]
#define GetMatchLength(index) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].lengths[(index) & UNIT_MASK]
#define SetMatchLink(index, link, length) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].links[(index) & UNIT_MASK] = (U32)(link)
#define SetMatchLength(index, link, length) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].lengths[(index) & UNIT_MASK] = (BYTE)(length)
#define SetMatchLinkAndLength(index, link, length) { size_t i_ = (index) >> UNIT_BITS, u_ = (index) & UNIT_MASK; ((RMF_unit*)tbl->table)[i_].links[u_] = (U32)(link); ((RMF_unit*)tbl->table)[i_].lengths[u_] = (BYTE)(length); }
#define SetNull(index) ((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].links[(index) & UNIT_MASK] = RADIX_NULL_LINK
#define IsNull(index) (((RMF_unit*)tbl->table)[(index) >> UNIT_BITS].links[(index) & UNIT_MASK] == RADIX_NULL_LINK)
BYTE* RMF_structuredAsOutputBuffer(FL2_matchTable* const tbl, size_t const index)
{
return (BYTE*)((RMF_unit*)tbl->table + (index >> UNIT_BITS) + ((index & UNIT_MASK) != 0));
}
/* Restrict the match lengths so that they don't reach beyond index */
void RMF_structuredLimitLengths(FL2_matchTable* const tbl, size_t const index)
{
DEBUGLOG(5, "RMF_limitLengths : end %u, max length %u", (U32)index, RADIX_MAX_LENGTH);
SetNull(index - 1);
for (size_t length = 2; length < RADIX_MAX_LENGTH && length <= index; ++length) {
size_t const i = (index - length) >> UNIT_BITS;
size_t const u = (index - length) & UNIT_MASK;
if (((RMF_unit*)tbl->table)[i].links[u] != RADIX_NULL_LINK) {
((RMF_unit*)tbl->table)[i].lengths[u] = MIN((BYTE)length, ((RMF_unit*)tbl->table)[i].lengths[u]);
}
}
}
#include "radix_engine.h"

101
C/fast-lzma2/range_enc.c Normal file
View File

@@ -0,0 +1,101 @@
/*
* Bitwise range encoder by Igor Pavlov
* Modified by Conor McCarthy
*
* Public domain
*/
#include "fl2_internal.h"
#include "mem.h"
#include "range_enc.h"
const unsigned price_table[kBitModelTotal >> kNumMoveReducingBits] = {
128, 103, 91, 84, 78, 73, 69, 66,
63, 61, 58, 56, 54, 52, 51, 49,
48, 46, 45, 44, 43, 42, 41, 40,
39, 38, 37, 36, 35, 34, 34, 33,
32, 31, 31, 30, 29, 29, 28, 28,
27, 26, 26, 25, 25, 24, 24, 23,
23, 22, 22, 22, 21, 21, 20, 20,
19, 19, 19, 18, 18, 17, 17, 17,
16, 16, 16, 15, 15, 15, 14, 14,
14, 13, 13, 13, 12, 12, 12, 11,
11, 11, 11, 10, 10, 10, 10, 9,
9, 9, 9, 8, 8, 8, 8, 7,
7, 7, 7, 6, 6, 6, 6, 5,
5, 5, 5, 5, 4, 4, 4, 4,
3, 3, 3, 3, 3, 2, 2, 2,
2, 2, 2, 1, 1, 1, 1, 1
};
void SetOutputBuffer(RangeEncoder* const rc, BYTE *const out_buffer, size_t chunk_size)
{
rc->out_buffer = out_buffer;
rc->chunk_size = chunk_size;
rc->out_index = 0;
}
void RangeEncReset(RangeEncoder* const rc)
{
rc->low = 0;
rc->range = (U32)-1;
rc->cache_size = 1;
rc->cache = 0;
}
void ShiftLow(RangeEncoder* const rc)
{
if (rc->low < 0xFF000000 || rc->low > 0xFFFFFFFF)
{
BYTE temp = rc->cache;
do {
assert (rc->out_index < rc->chunk_size - 4096);
rc->out_buffer[rc->out_index++] = temp + (BYTE)(rc->low >> 32);
temp = 0xFF;
} while (--rc->cache_size != 0);
rc->cache = (BYTE)(rc->low >> 24);
}
++rc->cache_size;
rc->low = (rc->low << 8) & 0xFFFFFFFF;
}
void EncodeBitTree(RangeEncoder* const rc, Probability *const probs, unsigned bit_count, unsigned symbol)
{
size_t tree_index = 1;
assert(bit_count > 0);
do {
unsigned bit;
--bit_count;
bit = (symbol >> bit_count) & 1;
EncodeBit(rc, &probs[tree_index], bit);
tree_index = (tree_index << 1) | bit;
} while (bit_count != 0);
}
void EncodeBitTreeReverse(RangeEncoder* const rc, Probability *const probs, unsigned bit_count, unsigned symbol)
{
unsigned tree_index = 1;
assert(bit_count != 0);
do {
unsigned bit = symbol & 1;
EncodeBit(rc, &probs[tree_index], bit);
tree_index = (tree_index << 1) + bit;
symbol >>= 1;
} while (--bit_count != 0);
}
void EncodeDirect(RangeEncoder* const rc, unsigned value, unsigned bit_count)
{
assert(bit_count > 0);
do {
rc->range >>= 1;
--bit_count;
rc->low += rc->range & -((int)(value >> bit_count) & 1);
if (rc->range < kTopValue) {
rc->range <<= 8;
ShiftLow(rc);
}
} while (bit_count != 0);
}

157
C/fast-lzma2/range_enc.h Normal file
View File

@@ -0,0 +1,157 @@
/*
* Bitwise range encoder by Igor Pavlov
* Modified by Conor McCarthy
*
* Public domain
*/
#ifndef RANGE_ENCODER_H
#define RANGE_ENCODER_H
#include "mem.h"
#include "compiler.h"
#if defined (__cplusplus)
extern "C" {
#endif
#ifdef LZMA_ENC_PROB32
typedef U32 Probability;
#else
typedef U16 Probability;
#endif
#define kNumTopBits 24U
#define kTopValue (1UL << kNumTopBits)
#define kNumBitModelTotalBits 11U
#define kBitModelTotal (1 << kNumBitModelTotalBits)
#define kNumMoveBits 5U
#define kProbInitValue (kBitModelTotal >> 1U)
#define kNumMoveReducingBits 4U
#define kNumBitPriceShiftBits 4U
extern const unsigned price_table[kBitModelTotal >> kNumMoveReducingBits];
typedef struct
{
BYTE *out_buffer;
size_t out_index;
size_t chunk_size;
U64 cache_size;
U64 low;
U32 range;
BYTE cache;
} RangeEncoder;
void RangeEncReset(RangeEncoder* const rc);
void SetOutputBuffer(RangeEncoder* const rc, BYTE *const out_buffer, size_t chunk_size);
void RangeEncReset(RangeEncoder* const rc);
void ShiftLow(RangeEncoder* const rc);
void EncodeBitTree(RangeEncoder* const rc, Probability *const probs, unsigned bit_count, unsigned symbol);
void EncodeBitTreeReverse(RangeEncoder* const rc, Probability *const probs, unsigned bit_count, unsigned symbol);
void EncodeDirect(RangeEncoder* const rc, unsigned value, unsigned bit_count);
HINT_INLINE
void EncodeBit0(RangeEncoder* const rc, Probability *const rprob)
{
unsigned prob = *rprob;
rc->range = (rc->range >> kNumBitModelTotalBits) * prob;
prob += (kBitModelTotal - prob) >> kNumMoveBits;
*rprob = (Probability)prob;
if (rc->range < kTopValue) {
rc->range <<= 8;
ShiftLow(rc);
}
}
HINT_INLINE
void EncodeBit1(RangeEncoder* const rc, Probability *const rprob)
{
unsigned prob = *rprob;
U32 new_bound = (rc->range >> kNumBitModelTotalBits) * prob;
rc->low += new_bound;
rc->range -= new_bound;
prob -= prob >> kNumMoveBits;
*rprob = (Probability)prob;
if (rc->range < kTopValue) {
rc->range <<= 8;
ShiftLow(rc);
}
}
HINT_INLINE
void EncodeBit(RangeEncoder* const rc, Probability *const rprob, unsigned const bit)
{
unsigned prob = *rprob;
if (bit != 0) {
U32 new_bound = (rc->range >> kNumBitModelTotalBits) * prob;
rc->low += new_bound;
rc->range -= new_bound;
prob -= prob >> kNumMoveBits;
}
else {
rc->range = (rc->range >> kNumBitModelTotalBits) * prob;
prob += (kBitModelTotal - prob) >> kNumMoveBits;
}
*rprob = (Probability)prob;
if (rc->range < kTopValue) {
rc->range <<= 8;
ShiftLow(rc);
}
}
#define GET_PRICE(rc, prob, symbol) \
price_table[((prob) ^ ((-(int)(symbol)) & (kBitModelTotal - 1))) >> kNumMoveReducingBits];
#define GET_PRICE_0(rc, prob) price_table[(prob) >> kNumMoveReducingBits]
#define GET_PRICE_1(rc, prob) price_table[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits]
HINT_INLINE
unsigned GetTreePrice(RangeEncoder* const rc, const Probability* const prob_table, unsigned const bit_count, size_t symbol)
{
unsigned price = 0;
symbol |= ((size_t)1 << bit_count);
while (symbol != 1) {
size_t next_symbol = symbol >> 1;
unsigned prob = prob_table[next_symbol];
unsigned bit = (unsigned)symbol & 1;
price += GET_PRICE(rc, prob, bit);
symbol = next_symbol;
}
return price;
}
HINT_INLINE
unsigned GetReverseTreePrice(RangeEncoder* const rc, const Probability* const prob_table, unsigned const bit_count, size_t symbol)
{
unsigned price = 0;
size_t m = 1;
for (unsigned i = bit_count; i != 0; --i) {
unsigned prob = prob_table[m];
unsigned bit = symbol & 1;
symbol >>= 1;
price += GET_PRICE(rc, prob, bit);
m = (m << 1) | bit;
}
return price;
}
HINT_INLINE
void Flush(RangeEncoder* const rc)
{
for (int i = 0; i < 5; ++i)
ShiftLow(rc);
}
#if defined (__cplusplus)
}
#endif
#endif /* RANGE_ENCODER_H */

765
C/fast-lzma2/util.h Normal file
View File

@@ -0,0 +1,765 @@
/*
* Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef UTIL_H_MODULE
#define UTIL_H_MODULE
#if defined (__cplusplus)
extern "C" {
#endif
/*-****************************************
* Dependencies
******************************************/
#include "platform.h" /* PLATFORM_POSIX_VERSION */
#include <stdlib.h> /* malloc */
#include <stddef.h> /* size_t, ptrdiff_t */
#include <stdio.h> /* fprintf */
#include <string.h> /* strncmp */
#include <sys/types.h> /* stat, utime */
#include <sys/stat.h> /* stat */
#if defined(_MSC_VER)
# include <sys/utime.h> /* utime */
# include <io.h> /* _chmod */
#else
# include <unistd.h> /* chown, stat */
# include <utime.h> /* utime */
#endif
#include <time.h> /* time */
#include <errno.h>
#include "mem.h" /* U32, U64 */
/* ************************************************************
* Avoid fseek()'s 2GiB barrier with MSVC, MacOS, *BSD, MinGW
***************************************************************/
#if defined(_MSC_VER) && (_MSC_VER >= 1400)
# define UTIL_fseek _fseeki64
#elif !defined(__64BIT__) && (PLATFORM_POSIX_VERSION >= 200112L) /* No point defining Large file for 64 bit */
# define UTIL_fseek fseeko
#elif defined(__MINGW32__) && defined(__MSVCRT__) && !defined(__STRICT_ANSI__) && !defined(__NO_MINGW_LFS)
# define UTIL_fseek fseeko64
#else
# define UTIL_fseek fseek
#endif
/*-****************************************
* Sleep functions: Windows - Posix - others
******************************************/
#if defined(_WIN32)
# include <windows.h>
# define SET_REALTIME_PRIORITY SetPriorityClass(GetCurrentProcess(), REALTIME_PRIORITY_CLASS)
# define UTIL_sleep(s) Sleep(1000*s)
# define UTIL_sleepMilli(milli) Sleep(milli)
#elif PLATFORM_POSIX_VERSION >= 0 /* Unix-like operating system */
# include <unistd.h>
# include <sys/resource.h> /* setpriority */
# include <time.h> /* clock_t, nanosleep, clock, CLOCKS_PER_SEC */
# if defined(PRIO_PROCESS)
# define SET_REALTIME_PRIORITY setpriority(PRIO_PROCESS, 0, -20)
# else
# define SET_REALTIME_PRIORITY /* disabled */
# endif
# define UTIL_sleep(s) sleep(s)
# if (defined(__linux__) && (PLATFORM_POSIX_VERSION >= 199309L)) || (PLATFORM_POSIX_VERSION >= 200112L) /* nanosleep requires POSIX.1-2001 */
# define UTIL_sleepMilli(milli) { struct timespec t; t.tv_sec=0; t.tv_nsec=milli*1000000ULL; nanosleep(&t, NULL); }
# else
# define UTIL_sleepMilli(milli) /* disabled */
# endif
#else
# define SET_REALTIME_PRIORITY /* disabled */
# define UTIL_sleep(s) /* disabled */
# define UTIL_sleepMilli(milli) /* disabled */
#endif
/* *************************************
* Constants
***************************************/
#define LIST_SIZE_INCREASE (8*1024)
/*-****************************************
* Compiler specifics
******************************************/
#if defined(__INTEL_COMPILER)
# pragma warning(disable : 177) /* disable: message #177: function was declared but never referenced, useful with UTIL_STATIC */
#endif
#if defined(__GNUC__)
# define UTIL_STATIC static __attribute__((unused))
#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
# define UTIL_STATIC static inline
#elif defined(_MSC_VER)
# define UTIL_STATIC static __inline
# pragma warning(disable : 4996) /* disable: C4996: 'strncpy': This function or variable may be unsafe. */
#else
# define UTIL_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */
#endif
/*-****************************************
* Console log
******************************************/
static int g_utilDisplayLevel;
#define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__)
#define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } }
/*-****************************************
* Time functions
******************************************/
#if defined(_WIN32) /* Windows */
typedef LARGE_INTEGER UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { UTIL_time_t x; QueryPerformanceCounter(&x); return x; }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static LARGE_INTEGER ticksPerSecond;
static int init = 0;
if (!init) {
if (!QueryPerformanceFrequency(&ticksPerSecond))
UTIL_DISPLAYLEVEL(1, "ERROR: QueryPerformanceFrequency() failure\n");
init = 1;
}
return 1000000000ULL*(clockEnd.QuadPart - clockStart.QuadPart)/ticksPerSecond.QuadPart;
}
#elif defined(__APPLE__) && defined(__MACH__)
#include <mach/mach_time.h>
typedef U64 UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return mach_absolute_time(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return (((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom))/1000ULL;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd)
{
static mach_timebase_info_data_t rate;
static int init = 0;
if (!init) {
mach_timebase_info(&rate);
init = 1;
}
return ((clockEnd - clockStart) * (U64)rate.numer) / ((U64)rate.denom);
}
#elif (PLATFORM_POSIX_VERSION >= 200112L)
#include <time.h>
typedef struct timespec UTIL_freq_t;
typedef struct timespec UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void)
{
UTIL_time_t time;
if (clock_gettime(CLOCK_MONOTONIC, &time))
UTIL_DISPLAYLEVEL(1, "ERROR: Failed to get time\n"); /* we could also exit() */
return time;
}
UTIL_STATIC UTIL_time_t UTIL_getSpanTime(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t diff;
if (end.tv_nsec < begin.tv_nsec) {
diff.tv_sec = (end.tv_sec - 1) - begin.tv_sec;
diff.tv_nsec = (end.tv_nsec + 1000000000ULL) - begin.tv_nsec;
} else {
diff.tv_sec = end.tv_sec - begin.tv_sec;
diff.tv_nsec = end.tv_nsec - begin.tv_nsec;
}
return diff;
}
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 micro = 0;
micro += 1000000ULL * diff.tv_sec;
micro += diff.tv_nsec / 1000ULL;
return micro;
}
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t begin, UTIL_time_t end)
{
UTIL_time_t const diff = UTIL_getSpanTime(begin, end);
U64 nano = 0;
nano += 1000000000ULL * diff.tv_sec;
nano += diff.tv_nsec;
return nano;
}
#else /* relies on standard C (note : clock_t measurements can be wrong when using multi-threading) */
typedef clock_t UTIL_time_t;
UTIL_STATIC UTIL_time_t UTIL_getTime(void) { return clock(); }
UTIL_STATIC U64 UTIL_getSpanTimeMicro(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
UTIL_STATIC U64 UTIL_getSpanTimeNano(UTIL_time_t clockStart, UTIL_time_t clockEnd) { return 1000000000ULL * (clockEnd - clockStart) / CLOCKS_PER_SEC; }
#endif
/* returns time span in microseconds */
UTIL_STATIC U64 UTIL_clockSpanMicro( UTIL_time_t clockStart )
{
UTIL_time_t const clockEnd = UTIL_getTime();
return UTIL_getSpanTimeMicro(clockStart, clockEnd);
}
UTIL_STATIC void UTIL_waitForNextTick(void)
{
UTIL_time_t const clockStart = UTIL_getTime();
UTIL_time_t clockEnd;
do {
clockEnd = UTIL_getTime();
} while (UTIL_getSpanTimeNano(clockStart, clockEnd) == 0);
}
/*-****************************************
* File functions
******************************************/
#if defined(_MSC_VER)
#define chmod _chmod
typedef struct __stat64 stat_t;
#else
typedef struct stat stat_t;
#endif
UTIL_STATIC int UTIL_setFileStat(const char *filename, stat_t *statbuf)
{
int res = 0;
struct utimbuf timebuf;
timebuf.actime = time(NULL);
timebuf.modtime = statbuf->st_mtime;
res += utime(filename, &timebuf); /* set access and modification times */
#if !defined(_WIN32)
res += chown(filename, statbuf->st_uid, statbuf->st_gid); /* Copy ownership */
#endif
res += chmod(filename, statbuf->st_mode & 07777); /* Copy file permissions */
errno = 0;
return -res; /* number of errors is returned */
}
UTIL_STATIC int UTIL_getFileStat(const char* infilename, stat_t *statbuf)
{
int r;
#if defined(_MSC_VER)
r = _stat64(infilename, statbuf);
if (r || !(statbuf->st_mode & S_IFREG)) return 0; /* No good... */
#else
r = stat(infilename, statbuf);
if (r || !S_ISREG(statbuf->st_mode)) return 0; /* No good... */
#endif
return 1;
}
UTIL_STATIC int UTIL_isRegularFile(const char* infilename)
{
stat_t statbuf;
return UTIL_getFileStat(infilename, &statbuf); /* Only need to know whether it is a regular file */
}
UTIL_STATIC U32 UTIL_isDirectory(const char* infilename)
{
int r;
stat_t statbuf;
#if defined(_MSC_VER)
r = _stat64(infilename, &statbuf);
if (!r && (statbuf.st_mode & _S_IFDIR)) return 1;
#else
r = stat(infilename, &statbuf);
if (!r && S_ISDIR(statbuf.st_mode)) return 1;
#endif
return 0;
}
UTIL_STATIC U32 UTIL_isLink(const char* infilename)
{
#if defined(_WIN32)
/* no symlinks on windows */
(void)infilename;
#else
int r;
stat_t statbuf;
r = lstat(infilename, &statbuf);
if (!r && S_ISLNK(statbuf.st_mode)) return 1;
#endif
return 0;
}
#define UTIL_FILESIZE_UNKNOWN ((U64)(-1))
UTIL_STATIC U64 UTIL_getFileSize(const char* infilename)
{
if (!UTIL_isRegularFile(infilename)) return UTIL_FILESIZE_UNKNOWN;
{ int r;
#if defined(_MSC_VER)
struct __stat64 statbuf;
r = _stat64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#elif defined(__MINGW32__) && defined (__MSVCRT__)
struct _stati64 statbuf;
r = _stati64(infilename, &statbuf);
if (r || !(statbuf.st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN;
#else
struct stat statbuf;
r = stat(infilename, &statbuf);
if (r || !S_ISREG(statbuf.st_mode)) return UTIL_FILESIZE_UNKNOWN;
#endif
return (U64)statbuf.st_size;
}
}
UTIL_STATIC U64 UTIL_getTotalFileSize(const char* const * const fileNamesTable, unsigned nbFiles)
{
U64 total = 0;
int error = 0;
unsigned n;
for (n=0; n<nbFiles; n++) {
U64 const size = UTIL_getFileSize(fileNamesTable[n]);
error |= (size == UTIL_FILESIZE_UNKNOWN);
total += size;
}
return error ? UTIL_FILESIZE_UNKNOWN : total;
}
/*
* A modified version of realloc().
* If UTIL_realloc() fails the original block is freed.
*/
UTIL_STATIC void *UTIL_realloc(void *ptr, size_t size)
{
void *newptr = realloc(ptr, size);
if (newptr) return newptr;
free(ptr);
return NULL;
}
#ifdef _WIN32
# define UTIL_HAS_CREATEFILELIST
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
WIN32_FIND_DATAA cFile;
HANDLE hFile;
dirLength = (int)strlen(dirName);
path = (char*) malloc(dirLength + 3);
if (!path) return 0;
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
path[dirLength+1] = '*';
path[dirLength+2] = 0;
hFile=FindFirstFileA(path, &cFile);
if (hFile == INVALID_HANDLE_VALUE) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName);
return 0;
}
free(path);
do {
fnameLength = (int)strlen(cFile.cFileName);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { FindClose(hFile); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '\\';
memcpy(path+dirLength+1, cFile.cFileName, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
if (strcmp (cFile.cFileName, "..") == 0 ||
strcmp (cFile.cFileName, ".") == 0) continue;
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
}
else if ((cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED)) {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; }
}
if (*bufStart + *pos + pathLength < *bufEnd) {
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
*pos += pathLength + 1;
nbFiles++;
}
}
free(path);
} while (FindNextFileA(hFile, &cFile));
FindClose(hFile);
return nbFiles;
}
#elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */
# define UTIL_HAS_CREATEFILELIST
# include <dirent.h> /* opendir, readdir */
# include <string.h> /* strerror, memcpy */
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
DIR *dir;
struct dirent *entry;
char* path;
int dirLength, fnameLength, pathLength, nbFiles = 0;
if (!(dir = opendir(dirName))) {
UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno));
return 0;
}
dirLength = (int)strlen(dirName);
errno = 0;
while ((entry = readdir(dir)) != NULL) {
if (strcmp (entry->d_name, "..") == 0 ||
strcmp (entry->d_name, ".") == 0) continue;
fnameLength = (int)strlen(entry->d_name);
path = (char*) malloc(dirLength + fnameLength + 2);
if (!path) { closedir(dir); return 0; }
memcpy(path, dirName, dirLength);
path[dirLength] = '/';
memcpy(path+dirLength+1, entry->d_name, fnameLength);
pathLength = dirLength+1+fnameLength;
path[pathLength] = 0;
if (!followLinks && UTIL_isLink(path)) {
UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path);
continue;
}
if (UTIL_isDirectory(path)) {
nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
} else {
if (*bufStart + *pos + pathLength >= *bufEnd) {
ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE;
*bufStart = (char*)UTIL_realloc(*bufStart, newListSize);
*bufEnd = *bufStart + newListSize;
if (*bufStart == NULL) { free(path); closedir(dir); return 0; }
}
if (*bufStart + *pos + pathLength < *bufEnd) {
strncpy(*bufStart + *pos, path, *bufEnd - (*bufStart + *pos));
*pos += pathLength + 1;
nbFiles++;
}
}
free(path);
errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */
}
if (errno != 0) {
UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s\n", dirName, strerror(errno));
free(*bufStart);
*bufStart = NULL;
}
closedir(dir);
return nbFiles;
}
#else
UTIL_STATIC int UTIL_prepareFileList(const char *dirName, char** bufStart, size_t* pos, char** bufEnd, int followLinks)
{
(void)bufStart; (void)bufEnd; (void)pos;
UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE)\n", dirName);
return 0;
}
#endif /* #ifdef _WIN32 */
/*
* UTIL_createFileList - takes a list of files and directories (params: inputNames, inputNamesNb), scans directories,
* and returns a new list of files (params: return value, allocatedBuffer, allocatedNamesNb).
* After finishing usage of the list the structures should be freed with UTIL_freeFileList(params: return value, allocatedBuffer)
* In case of error UTIL_createFileList returns NULL and UTIL_freeFileList should not be called.
*/
UTIL_STATIC const char** UTIL_createFileList(const char **inputNames, unsigned inputNamesNb, char** allocatedBuffer, unsigned* allocatedNamesNb, int followLinks)
{
size_t pos;
unsigned i, nbFiles;
char* buf = (char*)malloc(LIST_SIZE_INCREASE);
char* bufend = buf + LIST_SIZE_INCREASE;
const char** fileTable;
if (!buf) return NULL;
for (i=0, pos=0, nbFiles=0; i<inputNamesNb; i++) {
if (!UTIL_isDirectory(inputNames[i])) {
size_t const len = strlen(inputNames[i]);
if (buf + pos + len >= bufend) {
ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE;
buf = (char*)UTIL_realloc(buf, newListSize);
bufend = buf + newListSize;
if (!buf) return NULL;
}
if (buf + pos + len < bufend) {
strncpy(buf + pos, inputNames[i], bufend - (buf + pos));
pos += len + 1;
nbFiles++;
}
} else {
nbFiles += UTIL_prepareFileList(inputNames[i], &buf, &pos, &bufend, followLinks);
if (buf == NULL) return NULL;
} }
if (nbFiles == 0) { free(buf); return NULL; }
fileTable = (const char**)malloc((nbFiles+1) * sizeof(const char*));
if (!fileTable) { free(buf); return NULL; }
for (i=0, pos=0; i<nbFiles; i++) {
fileTable[i] = buf + pos;
pos += strlen(fileTable[i]) + 1;
}
if (buf + pos > bufend) { free(buf); free((void*)fileTable); return NULL; }
*allocatedBuffer = buf;
*allocatedNamesNb = nbFiles;
return fileTable;
}
UTIL_STATIC void UTIL_freeFileList(const char** filenameTable, char* allocatedBuffer)
{
if (allocatedBuffer) free(allocatedBuffer);
if (filenameTable) free((void*)filenameTable);
}
/* count the number of physical cores */
#if defined(_WIN32) || defined(WIN32)
#include <windows.h>
typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD);
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
{ LPFN_GLPI glpi;
BOOL done = FALSE;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL;
PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL;
DWORD returnLength = 0;
size_t byteOffset = 0;
glpi = (LPFN_GLPI)GetProcAddress(GetModuleHandle(TEXT("kernel32")),
"GetLogicalProcessorInformation");
if (glpi == NULL) {
goto failed;
}
while(!done) {
DWORD rc = glpi(buffer, &returnLength);
if (FALSE == rc) {
if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) {
if (buffer)
free(buffer);
buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength);
if (buffer == NULL) {
perror("zstd");
exit(1);
}
} else {
/* some other error */
goto failed;
}
} else {
done = TRUE;
}
}
ptr = buffer;
while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) {
if (ptr->Relationship == RelationProcessorCore) {
numPhysicalCores++;
}
ptr++;
byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION);
}
free(buffer);
return numPhysicalCores;
}
failed:
/* try to fall back on GetSystemInfo */
{ SYSTEM_INFO sysinfo;
GetSystemInfo(&sysinfo);
numPhysicalCores = sysinfo.dwNumberOfProcessors;
if (numPhysicalCores == 0) numPhysicalCores = 1; /* just in case */
}
return numPhysicalCores;
}
#elif defined(__APPLE__)
#include <sys/sysctl.h>
/* Use apple-provided syscall
* see: man 3 sysctl */
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static S32 numPhysicalCores = 0; /* apple specifies int32_t */
if (numPhysicalCores != 0) return numPhysicalCores;
{ size_t size = sizeof(S32);
int const ret = sysctlbyname("hw.physicalcpu", &numPhysicalCores, &size, NULL, 0);
if (ret != 0) {
if (errno == ENOENT) {
/* entry not present, fall back on 1 */
numPhysicalCores = 1;
} else {
perror("zstd: can't get number of physical cpus");
exit(1);
}
}
return numPhysicalCores;
}
}
#elif defined(__linux__)
/* parse /proc/cpuinfo
* siblings / cpu cores should give hyperthreading ratio
* otherwise fall back on sysconf */
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numPhysicalCores == -1) {
/* value not queryable, fall back on 1 */
return numPhysicalCores = 1;
}
/* try to determine if there's hyperthreading */
{ FILE* const cpuinfo = fopen("/proc/cpuinfo", "r");
#define BUF_SIZE 80
char buff[BUF_SIZE];
int siblings = 0;
int cpu_cores = 0;
int ratio = 1;
if (cpuinfo == NULL) {
/* fall back on the sysconf value */
return numPhysicalCores;
}
/* assume the cpu cores/siblings values will be constant across all
* present processors */
while (!feof(cpuinfo)) {
if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) {
if (strncmp(buff, "siblings", 8) == 0) {
const char* const sep = strchr(buff, ':');
if (*sep == '\0') {
/* formatting was broken? */
goto failed;
}
siblings = atoi(sep + 1);
}
if (strncmp(buff, "cpu cores", 9) == 0) {
const char* const sep = strchr(buff, ':');
if (*sep == '\0') {
/* formatting was broken? */
goto failed;
}
cpu_cores = atoi(sep + 1);
}
} else if (ferror(cpuinfo)) {
/* fall back on the sysconf value */
goto failed;
}
}
if (siblings && cpu_cores) {
ratio = siblings / cpu_cores;
}
failed:
fclose(cpuinfo);
return numPhysicalCores = numPhysicalCores / ratio;
}
}
#elif defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)
/* Use apple-provided syscall
* see: man 3 sysctl */
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
static int numPhysicalCores = 0;
if (numPhysicalCores != 0) return numPhysicalCores;
numPhysicalCores = (int)sysconf(_SC_NPROCESSORS_ONLN);
if (numPhysicalCores == -1) {
/* value not queryable, fall back on 1 */
return numPhysicalCores = 1;
}
return numPhysicalCores;
}
#else
UTIL_STATIC int UTIL_countPhysicalCores(void)
{
/* assume 1 */
return 1;
}
#endif
#if defined (__cplusplus)
}
#endif
#endif /* UTIL_H_MODULE */

8
CPP/7zip/7zip.mak
View File

@@ -33,6 +33,7 @@ OBJS = \
$(LZ5_OBJS) \
$(ZSTD_OBJS) \
$(ZSTDMT_OBJS) \
$(FASTLZMA2_OBJS) \
$(ASM_OBJS) \
$O\resource.res \
@@ -208,6 +209,11 @@ $(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c
$(COMPL_O2)
!ENDIF
!IFDEF FASTLZMA2_OBJS
$(FASTLZMA2_OBJS): ../../../../C/fast-lzma2/$(*B).c
$(COMPL_O2) -DNO_XXHASH
!ENDIF
!ELSE
@@ -287,6 +293,8 @@ $(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c
-I ../../../../C/lz4 \
-I ../../../../C/lz5 \
-I ../../../../C/zstd
{../../../../C/fast-lzma2}.c{$O}.obj::
$(COMPLB_O2) -DNO_XXHASH
!ENDIF

12
CPP/7zip/Bundles/Alone/makefile
View File

@@ -320,6 +320,18 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../UI/Console/Console.mak"
!include "../../Aes.mak"

14
CPP/7zip/Bundles/Format7z/makefile
View File

@@ -242,4 +242,18 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

12
CPP/7zip/Bundles/Format7zF/makefile
View File

@@ -117,4 +117,16 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

12
CPP/7zip/Bundles/Format7zFO/makefile
View File

@@ -118,4 +118,16 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

12
CPP/7zip/Bundles/Format7zUSB/makefile
View File

@@ -234,4 +234,16 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

165
CPP/7zip/Compress/Lzma2Encoder.cpp
View File

@@ -4,6 +4,8 @@
#include "../../../C/Alloc.h"
#include "../../../C/fast-lzma2/fl2_errors.h"
#include "../Common/CWrappers.h"
#include "../Common/StreamUtils.h"
@@ -119,4 +121,167 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream
return SResToHRESULT(res);
}
CFastEncoder::CFastEncoder()
{
_encoder = NULL;
reduceSize = 0;
}
CFastEncoder::~CFastEncoder()
{
if (_encoder)
FL2_freeCCtx(_encoder);
}
#define CHECK_F(f) if (FL2_isError(f)) return E_INVALIDARG; /* check and convert error code */
STDMETHODIMP CFastEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
for (UInt32 i = 0; i < numProps; i++)
{
RINOK(SetLzma2Prop(propIDs[i], coderProps[i], lzma2Props));
}
if (_encoder == NULL) {
_encoder = FL2_createCCtxMt(lzma2Props.numTotalThreads);
if (_encoder == NULL)
return E_OUTOFMEMORY;
}
if (lzma2Props.lzmaProps.algo > 2) {
if (lzma2Props.lzmaProps.algo > 3)
return E_INVALIDARG;
lzma2Props.lzmaProps.algo = 2;
FL2_CCtx_setParameter(_encoder, FL2_p_highCompression, 1);
FL2_CCtx_setParameter(_encoder, FL2_p_compressionLevel, lzma2Props.lzmaProps.level);
}
else {
FL2_CCtx_setParameter(_encoder, FL2_p_7zLevel, lzma2Props.lzmaProps.level);
}
dictSize = lzma2Props.lzmaProps.dictSize;
if (!dictSize) {
dictSize = (UInt32)1 << FL2_CCtx_setParameter(_encoder, FL2_p_dictionaryLog, 0);
}
reduceSize = lzma2Props.lzmaProps.reduceSize;
reduceSize += (reduceSize < (UInt64)-1); /* prevent extra buffer shift after read */
dictSize = (UInt32)min(dictSize, reduceSize);
unsigned dictLog = FL2_DICTLOG_MIN;
while (((UInt32)1 << dictLog) < dictSize)
++dictLog;
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_dictionaryLog, dictLog));
if (lzma2Props.lzmaProps.algo >= 0) {
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_strategy, (unsigned)lzma2Props.lzmaProps.algo));
}
if (lzma2Props.lzmaProps.fb > 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_fastLength, lzma2Props.lzmaProps.fb));
if (lzma2Props.lzmaProps.mc) {
unsigned ml = 0;
while (((UInt32)1 << ml) < lzma2Props.lzmaProps.mc)
++ml;
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_searchLog, ml));
}
if (lzma2Props.lzmaProps.lc >= 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_literalCtxBits, lzma2Props.lzmaProps.lc));
if (lzma2Props.lzmaProps.lp >= 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_literalPosBits, lzma2Props.lzmaProps.lp));
if (lzma2Props.lzmaProps.pb >= 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_posBits, lzma2Props.lzmaProps.pb));
FL2_CCtx_setParameter(_encoder, FL2_p_omitProperties, 1);
FL2_CCtx_setParameter(_encoder, FL2_p_doXXHash, 0);
return S_OK;
}
#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
STDMETHODIMP CFastEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
{
Byte prop;
unsigned i;
for (i = 0; i < 40; i++)
if (dictSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
break;
prop = (Byte)i;
return WriteStream(outStream, &prop, 1);
}
typedef struct
{
ISequentialOutStream* outStream;
ICompressProgressInfo* progress;
UInt64 in_processed;
UInt64 out_processed;
HRESULT res;
} EncodingObjects;
static int FL2LIB_CALL Progress(size_t done, void* opaque)
{
EncodingObjects* p = (EncodingObjects*)opaque;
if (p && p->progress) {
UInt64 in_processed = p->in_processed + done;
p->res = p->progress->SetRatioInfo(&in_processed, &p->out_processed);
return p->res != S_OK;
}
return 0;
}
static int FL2LIB_CALL Write(const void* src, size_t srcSize, void* opaque)
{
EncodingObjects* p = (EncodingObjects*)opaque;
p->res = WriteStream(p->outStream, src, srcSize);
return p->res != S_OK;
}
STDMETHODIMP CFastEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
{
HRESULT err = S_OK;
inBuffer.AllocAtLeast(dictSize);
EncodingObjects objs = { outStream, progress, 0, 0, S_OK };
FL2_blockBuffer block = { inBuffer, 0, 0, dictSize };
do
{
FL2_shiftBlock(_encoder, &block);
size_t inSize = dictSize - block.start;
err = ReadStream(inStream, inBuffer + block.start, &inSize);
if (err != S_OK)
break;
block.end += inSize;
if (inSize) {
size_t cSize = FL2_compressCCtxBlock_toFn(_encoder, Write, &objs, &block, Progress);
if (FL2_isError(cSize)) {
if (FL2_getErrorCode(cSize) == FL2_error_memory_allocation)
return E_OUTOFMEMORY;
return objs.res != S_OK ? objs.res : S_FALSE;
}
if (objs.res != S_OK)
return objs.res;
objs.out_processed += cSize;
objs.in_processed += inSize;
if (progress) {
err = progress->SetRatioInfo(&objs.in_processed, &objs.out_processed);
if (err != S_OK)
break;
}
if (block.end < dictSize)
break;
}
else break;
} while (err == S_OK);
if (err == S_OK) {
size_t cSize = FL2_endFrame_toFn(_encoder, Write, &objs);
if (FL2_isError(cSize))
return S_FALSE;
objs.out_processed += cSize;
err = objs.res;
}
return err;
}
}}

28
CPP/7zip/Compress/Lzma2Encoder.h
View File

@@ -4,8 +4,10 @@
#define __LZMA2_ENCODER_H
#include "../../../C/Lzma2Enc.h"
#include "../../../C/fast-lzma2/fast-lzma2.h"
#include "../../Common/MyCom.h"
#include "../../Common/MyBuffer.h"
#include "../ICoder.h"
@@ -37,6 +39,32 @@ public:
virtual ~CEncoder();
};
class CFastEncoder :
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public CMyUnknownImp
{
FL2_CCtx* _encoder;
CByteBuffer inBuffer;
UInt64 reduceSize;
UInt32 dictSize;
public:
MY_UNKNOWN_IMP3(
ICompressCoder,
ICompressSetCoderProperties,
ICompressWriteCoderProperties)
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
CFastEncoder();
virtual ~CFastEncoder();
};
}}
#endif

20
CPP/7zip/Compress/Lzma2Register.cpp
View File

@@ -14,9 +14,19 @@ namespace NCompress {
namespace NLzma2 {
REGISTER_CODEC_E(LZMA2,
CDecoder(),
CEncoder(),
0x21,
"LZMA2")
CDecoder(),
CEncoder(),
0x21,
"LZMA2")
}
}}
namespace NFLzma2 {
REGISTER_CODEC_E(FLZMA2,
NCompress::NLzma2::CDecoder(),
NCompress::NLzma2::CFastEncoder(),
0x21,
"FLZMA2")
}
}

108
CPP/7zip/UI/GUI/CompressDialog.cpp
View File

@@ -110,6 +110,7 @@ enum EMethodID
kLIZARD_M4,
kLZMA,
kLZMA2,
kFLZMA2,
kPPMd,
kBZip2,
kDeflate,
@@ -130,6 +131,7 @@ static LPCSTR const kMethodsLongnames[] =
, "Lizard, LIZv1 + Huffman"
, "LZMA"
, "LZMA2"
, "LZMA2, Fast"
, "PPMd"
, "BZip2"
, "Deflate"
@@ -150,6 +152,7 @@ static LPCSTR const kMethodsNames[] =
, "Lizard"
, "LZMA"
, "LZMA2"
, "FLZMA2"
, "PPMd"
, "BZip2"
, "Deflate"
@@ -195,6 +198,7 @@ static const EMethodID g_7zMethods[] =
kLIZARD_M2,
kLIZARD_M3,
kLIZARD_M4,
kFLZMA2,
kLZMA2,
kLZMA,
kPPMd,
@@ -209,6 +213,7 @@ static const EMethodID g_7zSfxMethods[] =
kZSTD,
kLZMA,
kLZMA2,
kFLZMA2,
kPPMd
};
@@ -978,6 +983,12 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam)
case IDC_COMPRESS_LEVEL:
{
{
const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
int index = FindRegistryFormatAlways(ai.Name);
NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
fo.ResetForLevelChange();
}
SetMethod(GetMethodID());
SetSolidBlockSize();
SetNumThreads();
@@ -1304,7 +1315,7 @@ void CCompressDialog::SetMethod(int keepMethodId)
}
}
if (!weUseSameMethod)
// if (!weUseSameMethod)
{
SetDictionary();
SetOrder();
@@ -1392,6 +1403,42 @@ void CCompressDialog::AddDictionarySize(UInt32 size)
m_Dictionary.SetItemData(index, size);
}
typedef enum {
FL2_fast,
FL2_opt,
FL2_ultra
} FL2_strategy;
typedef struct {
unsigned dictionaryLog; /* largest match distance : larger == more compression, more memory needed during decompression; >= 27 == more memory, slower */
unsigned overlapFraction; /* overlap between consecutive blocks in 1/16 units: larger == more compression, slower */
unsigned chainLog; /* fully searched segment : larger == more compression, slower, more memory; hybrid mode only (ultra) */
unsigned searchLog; /* nb of searches : larger == more compression, slower; hybrid mode only (ultra) */
unsigned searchDepth; /* maximum depth for resolving string matches : larger == more compression, slower; >= 64 == more memory, slower */
unsigned fastLength; /* acceptable match size for parser, not less than searchDepth : larger == more compression, slower; fast bytes parameter from 7-zip */
unsigned divideAndConquer; /* split long chains of 2-byte matches into shorter chains with a small overlap : faster, somewhat less compression; enabled by default */
unsigned bufferLog; /* buffer size for processing match chains is (dictionaryLog - bufferLog) : when divideAndConquer enabled, affects compression; */
/* when divideAndConquer disabled, affects speed in a hardware-dependent manner */
FL2_strategy strategy; /* encoder strategy : fast, optimized or ultra (hybrid) */
} FL2_compressionParameters;
#define FL2_MAX_7Z_CLEVEL 9
static const FL2_compressionParameters FL2_7zCParameters[FL2_MAX_7Z_CLEVEL + 1] = {
{ 0,0,0,0,0,0,0 },
{ 20, 1, 7, 0, 6, 32, 1, 8, FL2_fast }, /* 1 */
{ 20, 2, 7, 0, 12, 32, 1, 8, FL2_fast }, /* 2 */
{ 21, 2, 7, 0, 16, 32, 1, 8, FL2_fast }, /* 3 */
{ 20, 2, 7, 0, 16, 32, 1, 8, FL2_opt }, /* 4 */
{ 24, 2, 9, 0, 40, 48, 1, 8, FL2_ultra }, /* 5 */
{ 25, 2, 10, 0, 48, 64, 1, 8, FL2_ultra }, /* 6 */
{ 26, 2, 11, 1, 60, 96, 1, 9, FL2_ultra }, /* 7 */
{ 27, 2, 12, 2, 128, 128, 1, 10, FL2_ultra }, /* 8 */
{ 27, 3, 14, 3, 252, 160, 0, 10, FL2_ultra } /* 9 */
};
#define RMF_BUILDER_SIZE (8 * 0x40100U)
void CCompressDialog::SetDictionary()
{
m_Dictionary.ResetContent();
@@ -1458,6 +1505,39 @@ void CCompressDialog::SetDictionary()
break;
}
case kFLZMA2:
{
static const UInt32 kMinDicSize = (1 << 20);
level += !level;
if (level > FL2_MAX_7Z_CLEVEL)
level = FL2_MAX_7Z_CLEVEL;
if (defaultDict == (UInt32)(Int32)-1)
defaultDict = (UInt32)1 << FL2_7zCParameters[level].dictionaryLog;
m_Dictionary.SetCurSel(0);
for (unsigned i = 20; i <= 31; i++) {
UInt32 dict = (UInt32)1 << i;
if (dict >
#ifdef MY_CPU_64BIT
(1 << 30)
#else
(1 << 27)
#endif
)
continue;
AddDictionarySize(dict);
UInt64 decomprSize;
UInt64 requiredComprSize = GetMemoryUsage(dict, decomprSize);
if (dict <= defaultDict && (!maxRamSize_Defined || requiredComprSize <= maxRamSize))
m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1);
}
break;
}
case kPPMd:
{
if (defaultDict == (UInt32)(Int32)-1)
@@ -1598,9 +1678,14 @@ void CCompressDialog::SetOrder()
{
case kLZMA:
case kLZMA2:
case kFLZMA2:
{
if (defaultOrder == (UInt32)(Int32)-1)
defaultOrder = (level >= 7) ? 64 : 32;
if (defaultOrder == (UInt32)(Int32)-1) {
if (methodID == kFLZMA2)
defaultOrder = FL2_7zCParameters[level].fastLength;
else
defaultOrder = (level >= 7) ? 64 : 32;
}
for (unsigned i = 3; i <= 8; i++)
for (unsigned j = 0; j < 2; j++)
{
@@ -1820,6 +1905,7 @@ void CCompressDialog::SetNumThreads()
case kLIZARD_M4: numAlgoThreadsMax = 128; break;
case kLZMA: numAlgoThreadsMax = 2; break;
case kLZMA2: numAlgoThreadsMax = 32; break;
case kFLZMA2: numAlgoThreadsMax = 128; break;
case kBZip2: numAlgoThreadsMax = 32; break;
}
if (IsZipFormat())
@@ -1930,6 +2016,22 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory)
return size;
}
case kFLZMA2:
{
if (level > FL2_MAX_7Z_CLEVEL)
level = FL2_MAX_7Z_CLEVEL;
size += dict * 5 + (1UL << 18) * numThreads;
unsigned depth = FL2_7zCParameters[level].searchDepth;
UInt32 bufSize = UInt32(1) << (FL2_7zCParameters[level].dictionaryLog - FL2_7zCParameters[level].bufferLog);
size += (bufSize * 12 + RMF_BUILDER_SIZE) * numThreads;
if (dict > (UInt32(1) << 26) || depth > 63)
size += dict;
if (FL2_7zCParameters[level].strategy == FL2_ultra)
size += (UInt32(4) << 14) + (UInt32(4) << FL2_7zCParameters[level].chainLog);
decompressMemory = dict + (2 << 20);
return size;
}
case kPPMd:
{
decompressMemory = dict + (2 << 20);