xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-07 20:06:59 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #58 from conor42/master

Browse Source 
Add Fast LZMA2 codec
This commit is contained in:
Tino Reichardt
2018-11-05 17:29:02 +01:00
committed by GitHub
parent 71a295905f ab10047253
commit eff4d316fa
41 changed files with 9060 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
CPP/7zip/7zip.mak
View File

@@ -34,6 +34,7 @@ OBJS = \
$(LZ5_OBJS) \
$(ZSTD_OBJS) \
$(ZSTDMT_OBJS) \
$(FASTLZMA2_OBJS) \
$(ASM_OBJS) \
$O\resource.res \
@@ -209,6 +210,11 @@ $(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c
$(COMPL_O2)
!ENDIF
!IFDEF FASTLZMA2_OBJS
$(FASTLZMA2_OBJS): ../../../../C/fast-lzma2/$(*B).c
$(COMPL_O2) -DNO_XXHASH
!ENDIF
!ELSE
@@ -291,6 +297,8 @@ $(ZSTDMT_OBJS): ../../../../C/zstdmt/$(*B).c
-I ../../../../C/lz4 \
-I ../../../../C/lz5 \
-I ../../../../C/zstd
{../../../../C/fast-lzma2}.c{$O}.obj::
$(COMPLB_O2) -DNO_XXHASH
!ENDIF

12
CPP/7zip/Bundles/Alone/makefile
View File

@@ -320,6 +320,18 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../UI/Console/Console.mak"
!include "../../Aes.mak"

14
CPP/7zip/Bundles/Format7z/makefile
View File

@@ -242,4 +242,18 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

12
CPP/7zip/Bundles/Format7zF/makefile
View File

@@ -117,4 +117,16 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

12
CPP/7zip/Bundles/Format7zFO/makefile
View File

@@ -117,4 +117,16 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

12
CPP/7zip/Bundles/Format7zUSB/makefile
View File

@@ -234,4 +234,16 @@ ZSTDMT_OBJS = \
$O\lz5-mt_decompress.obj \
$O\zstd-mt_threading.obj \
FASTLZMA2_OBJS = \
$O\fl2_error_private.obj \
$O\fl2pool.obj \
$O\fl2threading.obj \
$O\fl2_common.obj \
$O\fl2_compress.obj \
$O\lzma2_enc.obj \
$O\radix_bitpack.obj \
$O\radix_mf.obj \
$O\radix_struct.obj \
$O\range_enc.obj \
!include "../../7zip.mak"

165
CPP/7zip/Compress/Lzma2Encoder.cpp
View File

@@ -4,6 +4,8 @@
#include "../../../C/Alloc.h"
#include "../../../C/fast-lzma2/fl2_errors.h"
#include "../Common/CWrappers.h"
#include "../Common/StreamUtils.h"
@@ -119,4 +121,167 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream
return SResToHRESULT(res);
}
CFastEncoder::CFastEncoder()
{
_encoder = NULL;
reduceSize = 0;
}
CFastEncoder::~CFastEncoder()
{
if (_encoder)
FL2_freeCCtx(_encoder);
}
#define CHECK_F(f) if (FL2_isError(f)) return E_INVALIDARG; /* check and convert error code */
STDMETHODIMP CFastEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
for (UInt32 i = 0; i < numProps; i++)
{
RINOK(SetLzma2Prop(propIDs[i], coderProps[i], lzma2Props));
}
if (_encoder == NULL) {
_encoder = FL2_createCCtxMt(lzma2Props.numTotalThreads);
if (_encoder == NULL)
return E_OUTOFMEMORY;
}
if (lzma2Props.lzmaProps.algo > 2) {
if (lzma2Props.lzmaProps.algo > 3)
return E_INVALIDARG;
lzma2Props.lzmaProps.algo = 2;
FL2_CCtx_setParameter(_encoder, FL2_p_highCompression, 1);
FL2_CCtx_setParameter(_encoder, FL2_p_compressionLevel, lzma2Props.lzmaProps.level);
}
else {
FL2_CCtx_setParameter(_encoder, FL2_p_7zLevel, lzma2Props.lzmaProps.level);
}
dictSize = lzma2Props.lzmaProps.dictSize;
if (!dictSize) {
dictSize = (UInt32)1 << FL2_CCtx_setParameter(_encoder, FL2_p_dictionaryLog, 0);
}
reduceSize = lzma2Props.lzmaProps.reduceSize;
reduceSize += (reduceSize < (UInt64)-1); /* prevent extra buffer shift after read */
dictSize = (UInt32)min(dictSize, reduceSize);
unsigned dictLog = FL2_DICTLOG_MIN;
while (((UInt32)1 << dictLog) < dictSize)
++dictLog;
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_dictionaryLog, dictLog));
if (lzma2Props.lzmaProps.algo >= 0) {
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_strategy, (unsigned)lzma2Props.lzmaProps.algo));
}
if (lzma2Props.lzmaProps.fb > 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_fastLength, lzma2Props.lzmaProps.fb));
if (lzma2Props.lzmaProps.mc) {
unsigned ml = 0;
while (((UInt32)1 << ml) < lzma2Props.lzmaProps.mc)
++ml;
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_searchLog, ml));
}
if (lzma2Props.lzmaProps.lc >= 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_literalCtxBits, lzma2Props.lzmaProps.lc));
if (lzma2Props.lzmaProps.lp >= 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_literalPosBits, lzma2Props.lzmaProps.lp));
if (lzma2Props.lzmaProps.pb >= 0)
CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_posBits, lzma2Props.lzmaProps.pb));
FL2_CCtx_setParameter(_encoder, FL2_p_omitProperties, 1);
FL2_CCtx_setParameter(_encoder, FL2_p_doXXHash, 0);
return S_OK;
}
#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
STDMETHODIMP CFastEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
{
Byte prop;
unsigned i;
for (i = 0; i < 40; i++)
if (dictSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
break;
prop = (Byte)i;
return WriteStream(outStream, &prop, 1);
}
typedef struct
{
ISequentialOutStream* outStream;
ICompressProgressInfo* progress;
UInt64 in_processed;
UInt64 out_processed;
HRESULT res;
} EncodingObjects;
static int FL2LIB_CALL Progress(size_t done, void* opaque)
{
EncodingObjects* p = (EncodingObjects*)opaque;
if (p && p->progress) {
UInt64 in_processed = p->in_processed + done;
p->res = p->progress->SetRatioInfo(&in_processed, &p->out_processed);
return p->res != S_OK;
}
return 0;
}
static int FL2LIB_CALL Write(const void* src, size_t srcSize, void* opaque)
{
EncodingObjects* p = (EncodingObjects*)opaque;
p->res = WriteStream(p->outStream, src, srcSize);
return p->res != S_OK;
}
STDMETHODIMP CFastEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
{
HRESULT err = S_OK;
inBuffer.AllocAtLeast(dictSize);
EncodingObjects objs = { outStream, progress, 0, 0, S_OK };
FL2_blockBuffer block = { inBuffer, 0, 0, dictSize };
do
{
FL2_shiftBlock(_encoder, &block);
size_t inSize = dictSize - block.start;
err = ReadStream(inStream, inBuffer + block.start, &inSize);
if (err != S_OK)
break;
block.end += inSize;
if (inSize) {
size_t cSize = FL2_compressCCtxBlock_toFn(_encoder, Write, &objs, &block, Progress);
if (FL2_isError(cSize)) {
if (FL2_getErrorCode(cSize) == FL2_error_memory_allocation)
return E_OUTOFMEMORY;
return objs.res != S_OK ? objs.res : S_FALSE;
}
if (objs.res != S_OK)
return objs.res;
objs.out_processed += cSize;
objs.in_processed += inSize;
if (progress) {
err = progress->SetRatioInfo(&objs.in_processed, &objs.out_processed);
if (err != S_OK)
break;
}
if (block.end < dictSize)
break;
}
else break;
} while (err == S_OK);
if (err == S_OK) {
size_t cSize = FL2_endFrame_toFn(_encoder, Write, &objs);
if (FL2_isError(cSize))
return S_FALSE;
objs.out_processed += cSize;
err = objs.res;
}
return err;
}
}}

28
CPP/7zip/Compress/Lzma2Encoder.h
View File

@@ -4,8 +4,10 @@
#define __LZMA2_ENCODER_H
#include "../../../C/Lzma2Enc.h"
#include "../../../C/fast-lzma2/fast-lzma2.h"
#include "../../Common/MyCom.h"
#include "../../Common/MyBuffer.h"
#include "../ICoder.h"
@@ -37,6 +39,32 @@ public:
virtual ~CEncoder();
};
class CFastEncoder :
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public CMyUnknownImp
{
FL2_CCtx* _encoder;
CByteBuffer inBuffer;
UInt64 reduceSize;
UInt32 dictSize;
public:
MY_UNKNOWN_IMP3(
ICompressCoder,
ICompressSetCoderProperties,
ICompressWriteCoderProperties)
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
CFastEncoder();
virtual ~CFastEncoder();
};
}}
#endif

20
CPP/7zip/Compress/Lzma2Register.cpp
View File

@@ -14,9 +14,19 @@ namespace NCompress {
namespace NLzma2 {
REGISTER_CODEC_E(LZMA2,
CDecoder(),
CEncoder(),
0x21,
"LZMA2")
CDecoder(),
CEncoder(),
0x21,
"LZMA2")
}
}}
namespace NFLzma2 {
REGISTER_CODEC_E(FLZMA2,
NCompress::NLzma2::CDecoder(),
NCompress::NLzma2::CFastEncoder(),
0x21,
"FLZMA2")
}
}

108
CPP/7zip/UI/GUI/CompressDialog.cpp
View File

@@ -110,6 +110,7 @@ enum EMethodID
kLIZARD_M4,
kLZMA,
kLZMA2,
kFLZMA2,
kPPMd,
kBZip2,
kDeflate,
@@ -130,6 +131,7 @@ static LPCSTR const kMethodsLongnames[] =
, "Lizard, LIZv1 + Huffman"
, "LZMA"
, "LZMA2"
, "LZMA2, Fast"
, "PPMd"
, "BZip2"
, "Deflate"
@@ -150,6 +152,7 @@ static LPCSTR const kMethodsNames[] =
, "Lizard"
, "LZMA"
, "LZMA2"
, "FLZMA2"
, "PPMd"
, "BZip2"
, "Deflate"
@@ -195,6 +198,7 @@ static const EMethodID g_7zMethods[] =
kLIZARD_M2,
kLIZARD_M3,
kLIZARD_M4,
kFLZMA2,
kLZMA2,
kLZMA,
kPPMd,
@@ -209,6 +213,7 @@ static const EMethodID g_7zSfxMethods[] =
kZSTD,
kLZMA,
kLZMA2,
kFLZMA2,
kPPMd
};
@@ -978,6 +983,12 @@ bool CCompressDialog::OnCommand(int code, int itemID, LPARAM lParam)
case IDC_COMPRESS_LEVEL:
{
{
const CArcInfoEx &ai = (*ArcFormats)[GetFormatIndex()];
int index = FindRegistryFormatAlways(ai.Name);
NCompression::CFormatOptions &fo = m_RegistryInfo.Formats[index];
fo.ResetForLevelChange();
}
SetMethod(GetMethodID());
SetSolidBlockSize();
SetNumThreads();
@@ -1304,7 +1315,7 @@ void CCompressDialog::SetMethod(int keepMethodId)
}
}
if (!weUseSameMethod)
// if (!weUseSameMethod)
{
SetDictionary();
SetOrder();
@@ -1392,6 +1403,42 @@ void CCompressDialog::AddDictionarySize(UInt32 size)
m_Dictionary.SetItemData(index, size);
}
typedef enum {
FL2_fast,
FL2_opt,
FL2_ultra
} FL2_strategy;
typedef struct {
unsigned dictionaryLog; /* largest match distance : larger == more compression, more memory needed during decompression; >= 27 == more memory, slower */
unsigned overlapFraction; /* overlap between consecutive blocks in 1/16 units: larger == more compression, slower */
unsigned chainLog; /* fully searched segment : larger == more compression, slower, more memory; hybrid mode only (ultra) */
unsigned searchLog; /* nb of searches : larger == more compression, slower; hybrid mode only (ultra) */
unsigned searchDepth; /* maximum depth for resolving string matches : larger == more compression, slower; >= 64 == more memory, slower */
unsigned fastLength; /* acceptable match size for parser, not less than searchDepth : larger == more compression, slower; fast bytes parameter from 7-zip */
unsigned divideAndConquer; /* split long chains of 2-byte matches into shorter chains with a small overlap : faster, somewhat less compression; enabled by default */
unsigned bufferLog; /* buffer size for processing match chains is (dictionaryLog - bufferLog) : when divideAndConquer enabled, affects compression; */
/* when divideAndConquer disabled, affects speed in a hardware-dependent manner */
FL2_strategy strategy; /* encoder strategy : fast, optimized or ultra (hybrid) */
} FL2_compressionParameters;
#define FL2_MAX_7Z_CLEVEL 9
static const FL2_compressionParameters FL2_7zCParameters[FL2_MAX_7Z_CLEVEL + 1] = {
{ 0,0,0,0,0,0,0 },
{ 20, 1, 7, 0, 6, 32, 1, 8, FL2_fast }, /* 1 */
{ 20, 2, 7, 0, 12, 32, 1, 8, FL2_fast }, /* 2 */
{ 21, 2, 7, 0, 16, 32, 1, 8, FL2_fast }, /* 3 */
{ 20, 2, 7, 0, 16, 32, 1, 8, FL2_opt }, /* 4 */
{ 24, 2, 9, 0, 40, 48, 1, 8, FL2_ultra }, /* 5 */
{ 25, 2, 10, 0, 48, 64, 1, 8, FL2_ultra }, /* 6 */
{ 26, 2, 11, 1, 60, 96, 1, 9, FL2_ultra }, /* 7 */
{ 27, 2, 12, 2, 128, 128, 1, 10, FL2_ultra }, /* 8 */
{ 27, 3, 14, 3, 252, 160, 0, 10, FL2_ultra } /* 9 */
};
#define RMF_BUILDER_SIZE (8 * 0x40100U)
void CCompressDialog::SetDictionary()
{
m_Dictionary.ResetContent();
@@ -1458,6 +1505,39 @@ void CCompressDialog::SetDictionary()
break;
}
case kFLZMA2:
{
static const UInt32 kMinDicSize = (1 << 20);
level += !level;
if (level > FL2_MAX_7Z_CLEVEL)
level = FL2_MAX_7Z_CLEVEL;
if (defaultDict == (UInt32)(Int32)-1)
defaultDict = (UInt32)1 << FL2_7zCParameters[level].dictionaryLog;
m_Dictionary.SetCurSel(0);
for (unsigned i = 20; i <= 31; i++) {
UInt32 dict = (UInt32)1 << i;
if (dict >
#ifdef MY_CPU_64BIT
(1 << 30)
#else
(1 << 27)
#endif
)
continue;
AddDictionarySize(dict);
UInt64 decomprSize;
UInt64 requiredComprSize = GetMemoryUsage(dict, decomprSize);
if (dict <= defaultDict && (!maxRamSize_Defined || requiredComprSize <= maxRamSize))
m_Dictionary.SetCurSel(m_Dictionary.GetCount() - 1);
}
break;
}
case kPPMd:
{
if (defaultDict == (UInt32)(Int32)-1)
@@ -1598,9 +1678,14 @@ void CCompressDialog::SetOrder()
{
case kLZMA:
case kLZMA2:
case kFLZMA2:
{
if (defaultOrder == (UInt32)(Int32)-1)
defaultOrder = (level >= 7) ? 64 : 32;
if (defaultOrder == (UInt32)(Int32)-1) {
if (methodID == kFLZMA2)
defaultOrder = FL2_7zCParameters[level].fastLength;
else
defaultOrder = (level >= 7) ? 64 : 32;
}
for (unsigned i = 3; i <= 8; i++)
for (unsigned j = 0; j < 2; j++)
{
@@ -1820,6 +1905,7 @@ void CCompressDialog::SetNumThreads()
case kLIZARD_M4: numAlgoThreadsMax = 128; break;
case kLZMA: numAlgoThreadsMax = 2; break;
case kLZMA2: numAlgoThreadsMax = 32; break;
case kFLZMA2: numAlgoThreadsMax = 128; break;
case kBZip2: numAlgoThreadsMax = 32; break;
}
if (IsZipFormat())
@@ -1930,6 +2016,22 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory)
return size;
}
case kFLZMA2:
{
if (level > FL2_MAX_7Z_CLEVEL)
level = FL2_MAX_7Z_CLEVEL;
size += dict * 5 + (1UL << 18) * numThreads;
unsigned depth = FL2_7zCParameters[level].searchDepth;
UInt32 bufSize = UInt32(1) << (FL2_7zCParameters[level].dictionaryLog - FL2_7zCParameters[level].bufferLog);
size += (bufSize * 12 + RMF_BUILDER_SIZE) * numThreads;
if (dict > (UInt32(1) << 26) || depth > 63)
size += dict;
if (FL2_7zCParameters[level].strategy == FL2_ultra)
size += (UInt32(4) << 14) + (UInt32(4) << FL2_7zCParameters[level].chainLog);
decompressMemory = dict + (2 << 20);
return size;
}
case kPPMd:
{
decompressMemory = dict + (2 << 20);