From 739ec50c75fbc4807514a7a7c631c630ebf8d3df Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Fri, 26 Oct 2018 00:09:25 +0200 Subject: [PATCH] Switch to Zstandard's New advanced API. - long distance matching is enabled - the compression should improve, the speed also - decompression code is single threaded only --- C/7zVersion.h | 4 +- C/7zVersionTr.h | 4 +- CPP/7zip/Bundles/Alone/makefile | 5 +- CPP/7zip/Bundles/Codec_zstd/makefile | 5 +- CPP/7zip/Bundles/Codec_zstdF/makefile | 5 +- CPP/7zip/Bundles/Format7z/makefile | 5 +- CPP/7zip/Bundles/Format7zF/makefile | 6 +- CPP/7zip/Bundles/Format7zFO/makefile | 3 - CPP/7zip/Bundles/Format7zUSB/makefile | 5 +- CPP/7zip/Bundles/SFXCon/makefile | 6 - CPP/7zip/Bundles/SFXWin/makefile | 6 - CPP/7zip/Compress/ZstdDecoder.cpp | 183 ++++++++++---------------- CPP/7zip/Compress/ZstdDecoder.h | 38 +++--- CPP/7zip/Compress/ZstdEncoder.cpp | 153 +++++++++++++-------- CPP/7zip/Compress/ZstdEncoder.h | 14 +- 15 files changed, 207 insertions(+), 235 deletions(-) diff --git a/C/7zVersion.h b/C/7zVersion.h index 422d8cc8..d4db5505 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ #define MY_VER_MAJOR 18 #define MY_VER_MINOR 05 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "18.05 ZS v1.3.7 R2" +#define MY_VERSION_NUMBERS "18.05 ZS v1.3.7 R3" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,7 +10,7 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2018-10-25" +#define MY_DATE "2018-10-28" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h index 99a793d4..a7a83afa 100644 --- a/C/7zVersionTr.h +++ b/C/7zVersionTr.h @@ -1,9 +1,9 @@ #define MY_VER_MAJOR 18 #define MY_VER_MINOR 05 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "1.3.7 R2" +#define MY_VERSION_NUMBERS "1.3.7 R3" #define MY_VERSION MY_VERSION_NUMBERS -#define MY_DATE "2018-10-25" +#define MY_DATE "2018-10-28" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Tino Reichardt" diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index f95c6bce..86d1d983 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -3,7 +3,7 @@ MY_CONSOLE = 1 CFLAGS = $(CFLAGS) !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -DZSTD_MULTITHREAD !ENDIF @@ -318,9 +318,6 @@ ZSTDMT_OBJS = \ $O\lz5-mt_common.obj \ $O\lz5-mt_compress.obj \ $O\lz5-mt_decompress.obj \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../UI/Console/Console.mak" diff --git a/CPP/7zip/Bundles/Codec_zstd/makefile b/CPP/7zip/Bundles/Codec_zstd/makefile index 47e93b60..f6fb820f 100644 --- a/CPP/7zip/Bundles/Codec_zstd/makefile +++ b/CPP/7zip/Bundles/Codec_zstd/makefile @@ -1,6 +1,6 @@ PROG = zstd-min.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID +CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_MULTITHREAD 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ @@ -44,9 +44,6 @@ ZSTD_OBJS = \ $O\zstd_opt.obj \ ZSTDMT_OBJS = \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Codec_zstdF/makefile b/CPP/7zip/Bundles/Codec_zstdF/makefile index 0ddfd93e..d8857707 100644 --- a/CPP/7zip/Bundles/Codec_zstdF/makefile +++ b/CPP/7zip/Bundles/Codec_zstdF/makefile @@ -1,6 +1,6 @@ PROG = zstd.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_LEGACY_SUPPORT +CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_LEGACY_SUPPORT -DZSTD_MULTITHREAD 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ @@ -51,9 +51,6 @@ ZSTD_OBJS = \ $O\zstd_v07.obj \ ZSTDMT_OBJS = \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 35a96149..2702daf6 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -1,6 +1,6 @@ PROG = 7za.dll DEF_FILE = ../../Archive/Archive2.def -CFLAGS = $(CFLAGS) +CFLAGS = $(CFLAGS) -DZSTD_LEGACY_SUPPORT -DZSTD_MULTITHREAD COMMON_OBJS = \ $O\CRC.obj \ @@ -240,9 +240,6 @@ ZSTDMT_OBJS = \ $O\lz5-mt_common.obj \ $O\lz5-mt_compress.obj \ $O\lz5-mt_decompress.obj \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zF/makefile b/CPP/7zip/Bundles/Format7zF/makefile index 69537076..9292325f 100644 --- a/CPP/7zip/Bundles/Format7zF/makefile +++ b/CPP/7zip/Bundles/Format7zF/makefile @@ -1,7 +1,6 @@ PROG = 7z.dll DEF_FILE = ../../Archive/Archive2.def -CFLAGS = $(CFLAGS) \ - -DEXTERNAL_CODECS -DZSTD_LEGACY_SUPPORT \ +CFLAGS = $(CFLAGS) -DEXTERNAL_CODECS -DZSTD_LEGACY_SUPPORT -DZSTD_MULTITHREAD !IFNDEF UNDER_CE CFLAGS = $(CFLAGS) -D_7ZIP_LARGE_PAGES @@ -116,9 +115,6 @@ ZSTDMT_OBJS = \ $O\lz5-mt_common.obj \ $O\lz5-mt_compress.obj \ $O\lz5-mt_decompress.obj \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zFO/makefile b/CPP/7zip/Bundles/Format7zFO/makefile index 3795c044..4efb869a 100644 --- a/CPP/7zip/Bundles/Format7zFO/makefile +++ b/CPP/7zip/Bundles/Format7zFO/makefile @@ -116,9 +116,6 @@ ZSTDMT_OBJS = \ $O\lz5-mt_common.obj \ $O\lz5-mt_compress.obj \ $O\lz5-mt_decompress.obj \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zUSB/makefile b/CPP/7zip/Bundles/Format7zUSB/makefile index 1519c837..da0d67f1 100644 --- a/CPP/7zip/Bundles/Format7zUSB/makefile +++ b/CPP/7zip/Bundles/Format7zUSB/makefile @@ -1,6 +1,6 @@ PROG = 7zu.dll DEF_FILE = ../../Archive/Archive2.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID +CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_MULTITHREAD COMMON_OBJS = \ $O\CRC.obj \ @@ -232,9 +232,6 @@ ZSTDMT_OBJS = \ $O\lz5-mt_common.obj \ $O\lz5-mt_compress.obj \ $O\lz5-mt_decompress.obj \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_compress.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index a310c81c..0b901a03 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -140,27 +140,21 @@ ZSTD_OBJS = \ $O\debug.obj \ $O\entropy_common.obj \ $O\error_private.obj \ - $O\fse_compress.obj \ $O\fse_decompress.obj \ $O\hist.obj \ - $O\huf_compress.obj \ $O\huf_decompress.obj \ $O\pool.obj \ $O\threading.obj \ $O\xxhash.obj \ $O\zstd_common.obj \ - $O\zstd_compress.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ $O\zstd_lazy.obj \ $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ $O\zstd_opt.obj \ ZSTDMT_OBJS = \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index 38304c62..3f16887e 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -159,27 +159,21 @@ ZSTD_OBJS = \ $O\debug.obj \ $O\entropy_common.obj \ $O\error_private.obj \ - $O\fse_compress.obj \ $O\fse_decompress.obj \ $O\hist.obj \ - $O\huf_compress.obj \ $O\huf_decompress.obj \ $O\pool.obj \ $O\threading.obj \ $O\xxhash.obj \ $O\zstd_common.obj \ - $O\zstd_compress.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ $O\zstd_lazy.obj \ $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ $O\zstd_opt.obj \ ZSTDMT_OBJS = \ - $O\zstd-mt_common.obj \ - $O\zstd-mt_decompress.obj \ $O\zstd-mt_threading.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp index d42bae8c..7ba80334 100644 --- a/CPP/7zip/Compress/ZstdDecoder.cpp +++ b/CPP/7zip/Compress/ZstdDecoder.cpp @@ -1,115 +1,53 @@ -// (C) 2016 Tino Reichardt +// (C) 2016 - 2018 Tino Reichardt #include "StdAfx.h" #include "ZstdDecoder.h" -int ZstdRead(void *arg, ZSTDCB_Buffer * in) -{ - struct ZstdStream *x = (struct ZstdStream*)arg; - size_t size = in->size; - - HRESULT res = ReadStream(x->inStream, in->buf, &size); - - /* catch errors */ - switch (res) { - case E_ABORT: - return -2; - case E_OUTOFMEMORY: - return -3; - } - - /* some other error -> read_fail */ - if (res != S_OK) - return -1; - - in->size = size; - *x->processedIn += size; - - return 0; -} - -int ZstdWrite(void *arg, ZSTDCB_Buffer * out) -{ - struct ZstdStream *x = (struct ZstdStream*)arg; - UInt32 todo = (UInt32)out->size; - UInt32 done = 0; - - while (todo != 0) - { - UInt32 block; - HRESULT res = x->outStream->Write((char*)out->buf + done, todo, &block); - - /* catch errors */ - switch (res) { - case E_ABORT: - return -2; - case E_OUTOFMEMORY: - return -3; - } - - done += block; - if (res == k_My_HRESULT_WritingWasCut) - break; - /* some other error -> write_fail */ - if (res != S_OK) - return -1; - - if (block == 0) - return -1; - todo -= block; - } - - *x->processedOut += done; - /* we need no lock here, cause only one thread can write... */ - if (x->progress) - x->progress->SetRatioInfo(x->processedIn, x->processedOut); - - return 0; -} - namespace NCompress { namespace NZSTD { CDecoder::CDecoder(): + _ctx(NULL), + _srcBuf(NULL), + _dstBuf(NULL), + _srcBufSize(ZSTD_DStreamInSize()), + _dstBufSize(ZSTD_DStreamOutSize()), _processedIn(0), _processedOut(0), - _inputSize(0), _numThreads(NWindows::NSystem::GetNumberOfProcessors()) + { _props.clear(); + _hMutex = CreateMutex(NULL, FALSE, NULL); } CDecoder::~CDecoder() { -} - -HRESULT CDecoder::ErrorOut(size_t code) -{ - const char *strError = ZSTDCB_getErrorString(code); - wchar_t wstrError[200+5]; /* no malloc here, /TR */ - - mbstowcs(wstrError, strError, 200); - MessageBoxW(0, wstrError, L"7-Zip Zstandard", MB_ICONERROR | MB_OK); - MyFree(wstrError); - - return S_FALSE; + if (_ctx) { + ZSTD_freeDCtx(_ctx); + MyFree(_srcBuf); + MyFree(_dstBuf); + CloseHandle(_hMutex); + } } STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) { DProps *pProps = (DProps *)prop; - if (size != sizeof(DProps)) + switch (size) { + case 3: + case 5: + memcpy(&_props, pProps, sizeof (DProps)); + return S_OK; + default: return E_NOTIMPL; - - memcpy(&_props, pProps, sizeof (DProps)); - - return S_OK; + } } STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) { - const UInt32 kNumThreadsMax = ZSTDCB_THREAD_MAX; + const UInt32 kNumThreadsMax = ZSTD_THREAD_MAX; if (numThreads < 1) numThreads = 1; if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; _numThreads = numThreads; @@ -132,42 +70,55 @@ STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize) HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress) { - ZSTDCB_RdWr_t rdwr; - size_t result; - HRESULT res = S_OK; + /* 1) create context */ + if (!_ctx) { + _ctx = ZSTD_createDCtx(); + if (!_ctx) + return E_OUTOFMEMORY; - struct ZstdStream Rd; - Rd.inStream = inStream; - Rd.processedIn = &_processedIn; + _srcBuf = MyAlloc(_srcBufSize); + if (!_srcBuf) + return E_OUTOFMEMORY; - struct ZstdStream Wr; - Wr.progress = progress; - Wr.outStream = outStream; - Wr.processedIn = &_processedIn; - Wr.processedOut = &_processedOut; - - /* 1) setup read/write functions */ - rdwr.fn_read = ::ZstdRead; - rdwr.fn_write = ::ZstdWrite; - rdwr.arg_read = (void *)&Rd; - rdwr.arg_write = (void *)&Wr; - - /* 2) create decompression context */ - ZSTDCB_DCtx *ctx = ZSTDCB_createDCtx(_numThreads, _inputSize); - if (!ctx) - return S_FALSE; - - /* 3) decompress */ - result = ZSTDCB_decompressDCtx(ctx, &rdwr); - if (ZSTDCB_isError(result)) { - if (result == (size_t)-ZSTDCB_error_canceled) - return E_ABORT; - return ErrorOut(result); + _dstBuf = MyAlloc(_dstBufSize); + if (!_dstBuf) + return E_OUTOFMEMORY; + } else { + ZSTD_resetDStream(_ctx); } - /* 4) free resources */ - ZSTDCB_freeDCtx(ctx); - return res; + for (;;) { + size_t size = _srcBufSize; + RINOK(ReadStream(inStream, _srcBuf, &size)); + for (;;) { + ZSTD_inBuffer inBuff = { _srcBuf, size, 0 }; + ZSTD_outBuffer outBuff= { _dstBuf, _dstBufSize, 0 }; + size_t const readSizeHint = ZSTD_decompressStream(_ctx, &outBuff, &inBuff); + + if (ZSTD_isError(readSizeHint)) + return E_FAIL; + + /* write decompressed data */ + RINOK(WriteStream(outStream, _dstBuf, outBuff.pos)); + WaitForSingleObject(_hMutex, INFINITE); + _processedOut += outBuff.pos; + RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); + ReleaseMutex(_hMutex); + + if (inBuff.pos > 0) { + memmove(_srcBuf, (char*)_srcBuf + inBuff.pos, inBuff.size - inBuff.pos); + size = _srcBufSize - inBuff.pos; + RINOK(ReadStream(inStream, (char*)_srcBuf + inBuff.pos, &size)); + } + + if (inBuff.size != inBuff.pos) + return E_FAIL; + + /* eof */ + if (readSizeHint == 0) + return S_OK; + } + } } STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream, diff --git a/CPP/7zip/Compress/ZstdDecoder.h b/CPP/7zip/Compress/ZstdDecoder.h index 175898a3..877a14db 100644 --- a/CPP/7zip/Compress/ZstdDecoder.h +++ b/CPP/7zip/Compress/ZstdDecoder.h @@ -1,10 +1,9 @@ -// (C) 2016 Tino Reichardt +// (C) 2016 - 2018 Tino Reichardt #define ZSTD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" #include "../../../C/zstd/zstd.h" -#include "../../../C/zstdmt/zstd-mt.h" #include "../../Windows/System.h" #include "../../Common/Common.h" @@ -14,16 +13,20 @@ #include "../Common/RegisterCodec.h" #include "../Common/ProgressMt.h" -struct ZstdStream { - ISequentialInStream *inStream; - ISequentialOutStream *outStream; - ICompressProgressInfo *progress; - UInt64 *processedIn; - UInt64 *processedOut; -}; +/** + * possible return values @ 7zip: + * S_OK / S_FALSE + * E_NOTIMPL + * E_NOINTERFACE + * E_ABORT + * E_FAIL + * E_OUTOFMEMORY + * E_INVALIDARG + */ -extern int ZstdRead(void *Stream, ZSTDCB_Buffer * in); -extern int ZstdWrite(void *Stream, ZSTDCB_Buffer * in); +#define ZSTD_LEVEL_MIN 1 +#define ZSTD_LEVEL_MAX 22 +#define ZSTD_THREAD_MAX 128 namespace NCompress { namespace NZSTD { @@ -36,7 +39,7 @@ struct DProps memset(this, 0, sizeof (*this)); _ver_major = ZSTD_VERSION_MAJOR; _ver_minor = ZSTD_VERSION_MINOR; - _level = 1; + _level = 3; } Byte _ver_major; @@ -51,18 +54,21 @@ class CDecoder:public ICompressCoder, public CMyUnknownImp { CMyComPtr < ISequentialInStream > _inStream; - DProps _props; + ZSTD_DCtx* _ctx; + void* _srcBuf; + void* _dstBuf; + size_t _srcBufSize; + size_t _dstBufSize; + UInt64 _processedIn; UInt64 _processedOut; - UInt32 _inputSize; UInt32 _numThreads; + HANDLE _hMutex; - HRESULT CDecoder::ErrorOut(size_t code); HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress); HRESULT SetOutStreamSizeResume(const UInt64 *outSize); - public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) diff --git a/CPP/7zip/Compress/ZstdEncoder.cpp b/CPP/7zip/Compress/ZstdEncoder.cpp index 23f55920..4958ff13 100644 --- a/CPP/7zip/Compress/ZstdEncoder.cpp +++ b/CPP/7zip/Compress/ZstdEncoder.cpp @@ -1,4 +1,10 @@ -// (C) 2016 Tino Reichardt +// (C) 2016 - 2018 Tino Reichardt + +#define DEBUG 0 + +#if DEBUG +#include +#endif #include "StdAfx.h" #include "ZstdEncoder.h" @@ -9,31 +15,27 @@ namespace NCompress { namespace NZSTD { CEncoder::CEncoder(): + _ctx(NULL), + _srcBuf(NULL), + _dstBuf(NULL), + _srcBufSize(ZSTD_CStreamInSize()), + _dstBufSize(ZSTD_CStreamOutSize()), _processedIn(0), _processedOut(0), - _inputSize(0), - _ctx(NULL), _numThreads(NWindows::NSystem::GetNumberOfProcessors()) { _props.clear(); + _hMutex = CreateMutex(NULL, FALSE, NULL); } CEncoder::~CEncoder() { - if (_ctx) - ZSTDCB_freeCCtx(_ctx); -} - -HRESULT CEncoder::ErrorOut(size_t code) -{ - const char *strError = ZSTDCB_getErrorString(code); - wchar_t wstrError[200+5]; /* no malloc here, /TR */ - - mbstowcs(wstrError, strError, 200); - MessageBoxW(0, wstrError, L"7-Zip Zstandard", MB_ICONERROR | MB_OK); - MyFree(wstrError); - - return S_FALSE; + if (_ctx) { + ZSTD_freeCCtx(_ctx); + MyFree(_srcBuf); + MyFree(_dstBuf); + CloseHandle(_hMutex); + } } STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps) @@ -54,7 +56,7 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI /* level 1..22 */ _props._level = static_cast < Byte > (prop.ulVal); - Byte mylevel = static_cast < Byte > (ZSTDCB_LEVEL_MAX); + Byte mylevel = static_cast < Byte > (ZSTD_LEVEL_MAX); if (_props._level > mylevel) _props._level = mylevel; @@ -84,52 +86,97 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 * /*inSize*/ , const UInt64 * /*outSize */, ICompressProgressInfo *progress) { - ZSTDCB_RdWr_t rdwr; - size_t result; - HRESULT res = S_OK; + ZSTD_EndDirective ZSTD_todo = ZSTD_e_continue; + ZSTD_outBuffer outBuff; + ZSTD_inBuffer inBuff; + size_t err, srcSize; - struct ZstdStream Rd; - Rd.inStream = inStream; - Rd.outStream = outStream; - Rd.processedIn = &_processedIn; - Rd.processedOut = &_processedOut; + if (!_ctx) { + _ctx = ZSTD_createCCtx(); + if (!_ctx) + return E_OUTOFMEMORY; - struct ZstdStream Wr; - if (_processedIn == 0) - Wr.progress = progress; - else - Wr.progress = 0; - Wr.inStream = inStream; - Wr.outStream = outStream; - Wr.processedIn = &_processedIn; - Wr.processedOut = &_processedOut; + _srcBuf = MyAlloc(_srcBufSize); + if (!_srcBuf) + return E_OUTOFMEMORY; - /* 1) setup read/write functions */ - rdwr.fn_read = ::ZstdRead; - rdwr.fn_write = ::ZstdWrite; - rdwr.arg_read = (void *)&Rd; - rdwr.arg_write = (void *)&Wr; + _dstBuf = MyAlloc(_dstBufSize); + if (!_dstBuf) + return E_OUTOFMEMORY; - /* 2) create compression context, if needed */ - if (!_ctx) - _ctx = ZSTDCB_createCCtx(_numThreads, _props._level, _inputSize); - if (!_ctx) - return S_FALSE; + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_compressionLevel, _props._level); + if (ZSTD_isError(err)) return E_FAIL; - /* 3) compress */ - result = ZSTDCB_compressCCtx(_ctx, &rdwr); - if (ZSTDCB_isError(result)) { - if (result == (size_t)-ZSTDCB_error_canceled) - return E_ABORT; - return ErrorOut(result); + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_nbWorkers, _numThreads); + if (ZSTD_isError(err)) return E_FAIL; + + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_contentSizeFlag, 1); + if (ZSTD_isError(err)) return E_FAIL; + + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_enableLongDistanceMatching, 1); + if (ZSTD_isError(err)) return E_FAIL; } - return res; + for (;;) { + + /* read input */ + srcSize = _srcBufSize; + RINOK(ReadStream(inStream, _srcBuf, &srcSize)); + + /* eof */ + if (srcSize == 0) + ZSTD_todo = ZSTD_e_end; + + /* compress data */ + WaitForSingleObject(_hMutex, INFINITE); + _processedIn += srcSize; + ReleaseMutex(_hMutex); + + for (;;) { + outBuff = { _dstBuf, _dstBufSize, 0 }; + + if (ZSTD_todo == ZSTD_e_continue) + inBuff = { _srcBuf, srcSize, 0 }; + else + inBuff = { NULL, srcSize, 0 }; + + err = ZSTD_compress_generic(_ctx, &outBuff, &inBuff, ZSTD_todo); + if (ZSTD_isError(err)) return E_FAIL; + +#if DEBUG + printf("err=%u ", (unsigned)err); + printf("srcSize=%u ", (unsigned)srcSize); + printf("todo=%u\n", ZSTD_todo); + printf("inBuff.size=%u ", (unsigned)inBuff.size); + printf("inBuff.pos=%u\n", (unsigned)inBuff.pos); + printf("outBuff.size=%u ", (unsigned)outBuff.size); + printf("outBuff.pos=%u\n\n", (unsigned)outBuff.pos); + fflush(stdout); +#endif + + /* write output */ + if (outBuff.pos) { + RINOK(WriteStream(outStream, _dstBuf, outBuff.pos)); + WaitForSingleObject(_hMutex, INFINITE); + _processedOut += outBuff.pos; + RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); + ReleaseMutex(_hMutex); + } + + /* done */ + if (ZSTD_todo == ZSTD_e_end && err == 0) + return S_OK; + + /* need more input */ + if (inBuff.pos == inBuff.size) + break; + } + } } STDMETHODIMP CEncoder::SetNumberOfThreads(UInt32 numThreads) { - const UInt32 kNumThreadsMax = ZSTDCB_THREAD_MAX; + const UInt32 kNumThreadsMax = ZSTD_THREAD_MAX; if (numThreads < 1) numThreads = 1; if (numThreads > kNumThreadsMax) numThreads = kNumThreadsMax; _numThreads = numThreads; diff --git a/CPP/7zip/Compress/ZstdEncoder.h b/CPP/7zip/Compress/ZstdEncoder.h index 7028fcb1..1f2c69a1 100644 --- a/CPP/7zip/Compress/ZstdEncoder.h +++ b/CPP/7zip/Compress/ZstdEncoder.h @@ -1,10 +1,9 @@ -// (C) 2016 Tino Reichardt +// (C) 2016 - 2018 Tino Reichardt #define ZSTD_STATIC_LINKING_ONLY #include "../../../C/Alloc.h" #include "../../../C/Threads.h" #include "../../../C/zstd/zstd.h" -#include "../../../C/zstdmt/zstd-mt.h" #include "../../Common/Common.h" #include "../../Common/MyCom.h" @@ -41,13 +40,16 @@ class CEncoder: { CProps _props; + ZSTD_CCtx* _ctx; + void* _srcBuf; + void* _dstBuf; + size_t _srcBufSize; + size_t _dstBufSize; + UInt64 _processedIn; UInt64 _processedOut; - UInt32 _inputSize; UInt32 _numThreads; - - ZSTDCB_CCtx *_ctx; - HRESULT CEncoder::ErrorOut(size_t code); + HANDLE _hMutex; public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder)