Switch from zbuff to stream interface

- using the recommended buffersizes from zstd lib
- compile also an codec dll with support for legacy versions

CPP/7zip/Bundles/Codec_zstd/makefile

@@ -16,7 +16,6 @@ C_OBJS = \
 ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
-  $O\zbuff_compress.obj \
   $O\zstd_compress.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Codec_zstdF/StdAfx.cpp

@@ -0,0 +1,3 @@
+// StdAfx.cpp
+
+#include "StdAfx.h"

CPP/7zip/Bundles/Codec_zstdF/StdAfx.h

@@ -0,0 +1,8 @@
+// StdAfx.h
+
+#ifndef __STDAFX_H
+#define __STDAFX_H
+
+#include "../../../Common/Common.h"
+
+#endif

CPP/7zip/Bundles/Codec_zstdF/makefile

@@ -0,0 +1,24 @@
+PROG = zstd.dll
+DEF_FILE = ../../Compress/Codec.def
+CFLAGS = $(CFLAGS) -DZSTD_LEGACY_SUPPORT
+
+7ZIP_COMMON_OBJS = \
+  $O\StreamUtils.obj \
+
+COMPRESS_OBJS = \
+  $O\CodecExports.obj \
+  $O\DllExportsCompress.obj \
+
+C_OBJS = \
+  $O\Alloc.obj \
+
+!include "../../zstd.mak"
+ZSTD_OBJS = $(ZSTD_OBJS) \
+  $O\fse_compress.obj \
+  $O\huf_compress.obj \
+  $O\zstd_compress.obj \
+  $O\zstd_v05.obj \
+  $O\zstd_v06.obj \
+  $O\zstd_v07.obj \
+
+!include "../../7zip.mak"

CPP/7zip/Bundles/Codec_zstdF/resource.rc

@@ -0,0 +1,5 @@
+#include "../../MyVersionInfo.rc"
+
+MY_VERSION_INFO_DLL("7z ZStandard Plugin", "zstd")
+
+101  ICON  "../../Archive/Icons/7z.ico"

CPP/7zip/Bundles/Format7z/makefile

@@ -1,8 +1,6 @@
 PROG = 7za.dll
 DEF_FILE = ../../Archive/Archive2.def
 CFLAGS = $(CFLAGS)
-#  -DDEFLATE_EXTRACT_ONLY \
-#  -DBZIP2_EXTRACT_ONLY \
 
 COMMON_OBJS = \
   $O\CRC.obj \
@@ -142,7 +140,6 @@ C_OBJS = \
 ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
-  $O\zbuff_compress.obj \
   $O\zstd_compress.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zF/makefile

@@ -1,7 +1,7 @@
 PROG = 7z.dll
 DEF_FILE = ../../Archive/Archive2.def
 CFLAGS = $(CFLAGS) \
-  -DEXTERNAL_CODECS -DZSTD_LEGACY \
+  -DEXTERNAL_CODECS -DZSTD_LEGACY_SUPPORT \
 
 !IFNDEF UNDER_CE
 CFLAGS = $(CFLAGS) -D_7ZIP_LARGE_PAGES
@@ -21,10 +21,9 @@ AR_OBJS = $(AR_OBJS) \
 ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
-  $O\zbuff_compress.obj \
   $O\zstd_compress.obj \
-  $O\zstd_v07.obj \
-  $O\zstd_v06.obj \
   $O\zstd_v05.obj \
+  $O\zstd_v06.obj \
+  $O\zstd_v07.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Compress/ZstdDecoder.cpp

@@ -1,84 +1,37 @@
 // ZstdDecoder.cpp
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 #include "ZstdDecoder.h"
 
-// #define SHOW_DEBUG_INFO
-#ifdef SHOW_DEBUG_INFO
-#include <stdio.h>
-#define PRF(x) x
-#else
-#define PRF(x)
-#endif
-
 namespace NCompress {
 namespace NZSTD {
 
 CDecoder::CDecoder():
-  _inBuf (NULL),
+  _dstream(NULL),
-  _outBuf (NULL),
+  _buffIn(NULL),
-  _inPos (0),
+  _buffOut(NULL),
-  _inSize (0),
+  _buffInSizeAllocated(0),
-  _eofFlag (false),
+  _buffOutSizeAllocated(0),
-  _state (NULL),
+  _buffInSize(ZSTD_DStreamInSize()),
-  _propsWereSet (false),
+  _buffOutSize(ZSTD_DStreamOutSize()*4),
-  _outSizeDefined (false),
+  _processedIn(0),
-  _outSize (0),
+  _processedOut(0),
-
+  _propsWereSet(false)
-  _inSizeProcessed (0),
-  _outSizeProcessed (0),
-  _inBufSizeAllocated (0),
-  _outBufSizeAllocated (0),
-
-  _inBufSize (ZBUFF_recommendedDInSize() * 30),
-  _outBufSize (ZBUFF_recommendedDOutSize()* 30)
 {
   _props.clear();
 }
 
 CDecoder::~CDecoder()
 {
-  if (_state)
+  if (_dstream)
-    ZB_freeDCtx(_state);
+    ZSTD_freeDStream(_dstream);
 
-  MyFree (_inBuf);
+  MyFree(_buffIn);
-  MyFree (_outBuf);
+  MyFree(_buffOut);
-}
 
-STDMETHODIMP CDecoder::SetInBufSize (UInt32, UInt32 size)
+  _buffInSizeAllocated = 0;
-{
+  _buffOutSizeAllocated = 0;
-  _inBufSize = size;
-  return S_OK;
-}
-
-STDMETHODIMP CDecoder::SetOutBufSize (UInt32, UInt32 size)
-{
-  _outBufSize = size;
-  return S_OK;
-}
-
-HRESULT CDecoder::CreateBuffers ()
-{
-  if (_inBuf == 0 || _inBufSize != _inBufSizeAllocated)
-  {
-    MyFree (_inBuf);
-    _inBuf = (Byte *) MyAlloc (_inBufSize);
-    if (_inBuf == 0)
-      return E_OUTOFMEMORY;
-    _inBufSizeAllocated = (UInt32)_inBufSize;
-  }
-
-  if (_outBuf == 0 || _outBufSize != _outBufSizeAllocated)
-  {
-    MyFree (_outBuf);
-    _outBuf = (Byte *) MyAlloc (_outBufSize);
-    if (_outBuf == 0)
-      return E_OUTOFMEMORY;
-    _outBufSizeAllocated = (UInt32)_outBufSize;
-  }
-
-  return S_OK;
 }
 
 STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
@@ -88,57 +41,86 @@ STDMETHODIMP CDecoder::SetDecoderProperties2 (const Byte * prop, UInt32 size)
   if (size != sizeof(DProps))
     return E_FAIL;
 
-  // version 0.x currently
+#ifdef ZSTD_LEGACY_SUPPORT
-  if (pProps->_ver_major != ZSTD_VERSION_MAJOR)
+  /* version 0.x and 1.x are okay */
+  if (pProps->_ver_major > 1)
     return E_FAIL;
 
+  /* 0.5, 0.6, 0.7, 0.8 are supported! */
+  if (pProps->_ver_major == 0) {
     switch (pProps->_ver_minor) {
-  case ZSTD_VERSION_MINOR:
-    break;
-#ifdef ZSTD_LEGACY
     case 5:
       break;
     case 6:
       break;
     case 7:
       break;
-#endif
+    case 8:
+      break;
     default:
       return E_FAIL;
-  }
+  }}
+#else
+  /* only exact version is okay */
+  if (pProps->_ver_major != ZSTD_VERSION_MAJOR)
+    return E_FAIL;
+  if (pProps->_ver_minor != ZSTD_VERSION_MINOR)
+    return E_FAIL;
+
+#endif
 
   memcpy(&_props, pProps, sizeof (DProps));
   _propsWereSet = true;
 
-  return CreateBuffers();
+  return S_OK;
 }
 
 HRESULT CDecoder::CreateDecompressor()
 {
+  size_t result;
+
   if (!_propsWereSet)
     return E_FAIL;
 
-  if (!_state) {
+  if (!_dstream) {
-    _state = ZB_createDCtx();
+    _dstream = ZSTD_createDStream();
-    if (!_state)
+    if (!_dstream)
       return E_FAIL;
   }
 
-  if (ZBUFF_isError(ZB_decompressInit(_state)))
+  result = ZSTD_initDStream(_dstream);
+  if (ZSTD_isError(result))
     return E_FAIL;
 
-  _eofFlag = false;
+  /* allocate buffers */
+  if (_buffInSizeAllocated != _buffInSize)
+  {
+    if (_buffIn)
+      MyFree(_buffIn);
+    _buffIn = MyAlloc(_buffInSize);
+
+    if (!_buffIn)
+      return E_OUTOFMEMORY;
+    _buffInSizeAllocated = _buffInSize;
+  }
+
+  if (_buffOutSizeAllocated != _buffOutSize)
+  {
+    if (_buffOut)
+      MyFree(_buffOut);
+    _buffOut = MyAlloc(_buffOutSize);
+
+    if (!_buffOut)
+      return E_OUTOFMEMORY;
+    _buffOutSizeAllocated = _buffOutSize;
+  }
 
   return S_OK;
 }
 
-HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * outSize)
+HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
 {
-  _outSizeDefined = (outSize != NULL);
+  _processedOut = 0;
-  if (_outSizeDefined)
-    _outSize = *outSize;
-  _outSizeProcessed = 0;
-
   RINOK(CreateDecompressor());
 
   return S_OK;
@@ -146,184 +128,71 @@ HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * outSize)
 
 STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
 {
-  _inSizeProcessed = 0;
+  _processedIn = 0;
-  _inPos = _inSize = 0;
   RINOK(SetOutStreamSizeResume(outSize));
+
   return S_OK;
 }
 
-HRESULT CDecoder::CodeSpec (ISequentialInStream * inStream,
+STDMETHODIMP CDecoder::SetInBufSize(UInt32, UInt32 size)
-		      ISequentialOutStream * outStream,
-		      ICompressProgressInfo * progress)
 {
-  if (_inBuf == 0 || !_propsWereSet)
+  _buffInSize = size;
-    return S_FALSE;
-
-  if (!_state)
-  {
-    if (CreateDecompressor () != S_OK)
-      return E_FAIL;
-  }
-
-  UInt64 startInProgress = _inSizeProcessed;
-
-  for (;;)
-  {
-    if ((!_eofFlag) && (_inPos == _inSize))
-    {
-      _inPos = _inSize = 0;
-      RINOK (inStream->Read (_inBuf, _inBufSizeAllocated, &_inSize));
-      if (!_inSize)
-	_eofFlag = true;
-    }
-
-    Byte *pIn_bytes = _inBuf + _inPos;
-    size_t num_in_bytes = _inSize - _inPos;
-    Byte *pOut_bytes = _outBuf;
-    size_t num_out_bytes = _outBufSize;
-    if (_outSizeDefined)
-    {
-      UInt64 out_remaining = _outSize - _outSizeProcessed;
-      if (out_remaining == 0)
   return S_OK;
-      if (num_out_bytes > out_remaining)
-	num_out_bytes = static_cast < size_t > (out_remaining);
 }
 
-    size_t decomp_status =
+STDMETHODIMP CDecoder::SetOutBufSize(UInt32, UInt32 size)
-      ZB_decompressContinue (_state, pOut_bytes, &num_out_bytes,
-				pIn_bytes, &num_in_bytes);
-    bool decomp_finished = (decomp_status == 0);
-    bool decomp_failed = ZBUFF_isError (decomp_status) != 0;
-
-    if (num_in_bytes)
 {
-      _inPos += (UInt32) num_in_bytes;
+  _buffOutSize = size;
-      _inSizeProcessed += (UInt32) num_in_bytes;
+  return S_OK;
 }
 
-    if (num_out_bytes)
+HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress)
 {
-      _outSizeProcessed += num_out_bytes;
+  RINOK(CreateDecompressor());
 
-      RINOK (WriteStream (outStream, _outBuf, num_out_bytes));
+  size_t result;
-    }
+  UInt32 const toRead = static_cast < const UInt32 > (_buffInSize);
+  for(;;) {
+    UInt32 read;
 
-    if (decomp_failed)
+    /* read input */
-    {
+    RINOK(inStream->Read(_buffIn, toRead, &read));
-      PRF(fprintf(stderr, "zstdcodec: ZB_decompressContinue() failed: %s\n", ZBUFF_getErrorName(decomp_status)));
+    size_t InSize = static_cast < size_t > (read);
+    _processedIn += InSize;
+
+    if (InSize == 0)
+      return S_OK;
+
+    /* decompress input */
+    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
+    for (;;) {
+      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+      result = ZSTD_decompressStream(_dstream, &output , &input);
+      if (ZSTD_isError(result))
         return S_FALSE;
-    }
+      /* write decompressed stream and update progress */
+      RINOK(WriteStream(outStream, _buffOut, output.pos));
+      _processedOut += output.pos;
+      RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
 
-    if (decomp_finished)
+      /* one more round */
+      if ((input.pos == input.size) && (result == 1))
+        continue;
+
+      /* finished */
+      if (input.pos == input.size)
         break;
+    }
+  }
+}
 
-    // This check is to prevent locking up if the input file is accidently truncated.
+STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
-    bool made_forward_progress = (num_out_bytes != 0) || (num_in_bytes != 0);
+  const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress)
-    if ((!made_forward_progress) && (_eofFlag))
 {
-      return S_FALSE;
-    }
-
-    UInt64 inSize = _inSizeProcessed - startInProgress;
-    if (progress)
-    {
-      RINOK (progress->SetRatioInfo (&inSize, &_outSizeProcessed));
-    }
-  }
-
-  return S_OK;
-}
-
-STDMETHODIMP CDecoder::Code (ISequentialInStream * inStream,
-		  ISequentialOutStream * outStream,
-		  const UInt64 * inSize,
-		  const UInt64 * outSize, ICompressProgressInfo * progress)
-{
-  (void) inSize;
-  if (_inBuf == 0)
-    return E_INVALIDARG;
   SetOutStreamSize(outSize);
   return CodeSpec(inStream, outStream, progress);
 }
 
-// wrapper for different versions
-void *CDecoder::ZB_createDCtx(void)
-{
-  PRF(fprintf(stderr, "zstdcodec: ZB_createDCtx(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return (void*)ZBUFFv05_createDCtx();
-    break;
-  case 6:
-    return (void*)ZBUFFv06_createDCtx();
-    break;
-  case 7:
-    return (void*)ZBUFFv07_createDCtx();
-    break;
-  }
-#endif
-  return (void*)ZBUFF_createDCtx();
-}
-
-size_t CDecoder::ZB_freeDCtx(void *dctx)
-{
-  PRF(fprintf(stderr, "zstdcodec: ZB_freeDCtx(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx *)dctx);
-    break;
-  case 6:
-    return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx *)dctx);
-    break;
-  case 7:
-    return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx *)dctx);
-    break;
-  }
-#endif
-  return ZBUFF_freeDCtx((ZBUFF_DCtx *)dctx);
-}
-
-size_t CDecoder::ZB_decompressInit(void *dctx)
-{
-  PRF(fprintf(stderr, "zstdcodec: ZB_decompressInit(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return ZBUFFv05_decompressInit((ZBUFFv05_DCtx *)dctx);
-    break;
-  case 6:
-    return ZBUFFv06_decompressInit((ZBUFFv06_DCtx *)dctx);
-    break;
-  case 7:
-    return ZBUFFv07_decompressInit((ZBUFFv07_DCtx *)dctx);
-    break;
-  }
-#endif
-  return ZBUFF_decompressInit((ZBUFF_DCtx *)dctx);
-}
-
-size_t CDecoder::ZB_decompressContinue(void *dctx, void* dst, size_t *dstCapacityPtr, const void* src, size_t *srcSizePtr)
-{
-  PRF(fprintf(stderr, "zstdcodec: ZB_decompressContinue(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return ZBUFFv05_decompressContinue((ZBUFFv05_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-    break;
-  case 6:
-    return ZBUFFv06_decompressContinue((ZBUFFv06_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-    break;
-  case 7:
-    return ZBUFFv07_decompressContinue((ZBUFFv07_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-    break;
-  }
-#endif
-  return ZBUFF_decompressContinue((ZBUFF_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-}
-
 #ifndef NO_READ_FROM_CODER
 STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
 {
@@ -334,82 +203,58 @@ STDMETHODIMP CDecoder::SetInStream (ISequentialInStream * inStream)
 STDMETHODIMP CDecoder::ReleaseInStream()
 {
   _inStream.Release();
+
   return S_OK;
 }
 
-STDMETHODIMP CDecoder::Read (void *data, UInt32 size, UInt32 * processedSize)
+STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize)
 {
   if (processedSize)
     *processedSize = 0;
 
-  if (_inBuf == 0 || !_propsWereSet)
+  size_t result;
-    return S_FALSE;
 
-  if (!_state)
+  if (!_dstream)
-  {
     if (CreateDecompressor() != S_OK)
       return E_FAIL;
-  }
 
-  while (size != 0)
+  UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
-  {
+  Byte *dataout = static_cast < Byte* > (data);
-    if ((!_eofFlag) && (_inPos == _inSize))
+  for(;;) {
-    {
+    /* read input */
-      _inPos = _inSize = 0;
+    RINOK(_inStream->Read(_buffIn, toRead, &read));
-      RINOK (_inStream->Read (_inBuf, _inBufSizeAllocated, &_inSize));
+    size_t InSize = static_cast < size_t > (read);
-      if (!_inSize)
+    _processedIn += InSize;
-	_eofFlag = true;
-    }
-
-    Byte *pIn_bytes = _inBuf + _inPos;
-    size_t  num_in_bytes = _inSize - _inPos;
-    Byte *pOut_bytes = (Byte *) data;
-    size_t  num_out_bytes = size;
-
-    size_t decomp_status =
-      ZB_decompressContinue (_state, pOut_bytes, &num_out_bytes,
-				pIn_bytes, &num_in_bytes);
-    bool
-      decomp_finished = (decomp_status == 0);
-    bool
-      decomp_failed = ZBUFF_isError (decomp_status) != 0;
-
-    if (num_in_bytes)
-    {
-      _inPos += (UInt32) num_in_bytes;
-      _inSizeProcessed += num_in_bytes;
-    }
-
-    if (num_out_bytes)
-    {
-      _outSizeProcessed += num_out_bytes;
-      size -= (UInt32) num_out_bytes;
-      if (processedSize)
-	*processedSize += (UInt32) num_out_bytes;
-    }
-
-    if (decomp_failed)
-    {
-      return S_FALSE;
-    }
-
-    if (decomp_finished)
-      break;
-
-    // This check is to prevent locking up if the input file is accidently truncated.
-    bool made_forward_progress = (num_out_bytes != 0) || (num_in_bytes != 0);
-    if ((!made_forward_progress) && (_eofFlag))
-    {
-      return S_FALSE;
-    }
-  }
 
+    if (InSize == 0) {
       return S_OK;
     }
 
-HRESULT CDecoder::CodeResume (ISequentialOutStream * outStream,
+    /* decompress input */
-			const UInt64 * outSize,
+    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
-			ICompressProgressInfo * progress)
+    for (;;) {
+      ZSTD_outBuffer output = { dataout, _buffOutSize, 0 };
+      result = ZSTD_decompressStream(_dstream, &output , &input);
+      if (ZSTD_isError(result))
+        return S_FALSE;
+
+      if (processedSize)
+        *processedSize += static_cast < UInt32 > (output.pos);
+
+      dataout += output.pos;
+
+      /* one more round */
+      if ((input.pos == input.size) && (result == 1))
+        continue;
+
+      /* finished */
+      if (input.pos == input.size)
+        break;
+    }
+  }
+}
+
+HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
 {
   RINOK(SetOutStreamSizeResume(outSize));
   return CodeSpec(_inStream, outStream, progress);

CPP/7zip/Compress/ZstdDecoder.h

@@ -1,11 +1,16 @@
 // ZstdDecoder.h
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
+
+/**
+ * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible
+ * with these versions: 0.5, 0.6, 0.7, 0.8 (0.8 == 1.0)
+ *
+ * /TR 2016-09-04
+ */
 
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../../../C/Alloc.h"
 #include "../../../C/ZStd/zstd.h"
-#include "../../../C/ZStd/zbuff.h"
-#include "../../../C/ZStd/zstd_legacy.h"
 
 #include "../../Common/Common.h"
 #include "../../Common/MyCom.h"
@@ -42,37 +47,25 @@ class CDecoder:public ICompressCoder,
   public CMyUnknownImp
 {
   CMyComPtr < ISequentialInStream > _inStream;
-  Byte *_inBuf;
-  Byte *_outBuf;
-  UInt32 _inPos;
-  UInt32 _inSize;
-  bool _eofFlag;
-
-  void *_state;
 
   DProps _props;
   bool _propsWereSet;
 
-  bool _outSizeDefined;
+  ZSTD_DStream *_dstream;
-  UInt64 _outSize;
+  void *_buffIn;
-  UInt64 _inSizeProcessed;
+  void *_buffOut;
-  UInt64 _outSizeProcessed;
 
-  UInt32 _inBufSizeAllocated;
+  size_t _buffInSize;
-  UInt32 _outBufSizeAllocated;
+  size_t _buffOutSize;
-  size_t _inBufSize;
+  size_t _buffInSizeAllocated;
-  size_t _outBufSize;
+  size_t _buffOutSizeAllocated;
 
-  HRESULT CreateBuffers ();
+  UInt64 _processedIn;
+  UInt64 _processedOut;
+
+  HRESULT CDecoder::CreateDecompressor();
   HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
   HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
-  HRESULT CreateDecompressor ();
-
-  // wrapper for different versions
-  void *ZB_createDCtx(void);
-  size_t ZB_freeDCtx(void *dctx);
-  size_t ZB_decompressInit(void *dctx);
-  size_t ZB_decompressContinue(void *dctx, void* dst, size_t *dstCapacityPtr, const void* src, size_t *srcSizePtr);
 
 public:
 
@@ -98,8 +91,7 @@ public:
   STDMETHOD (ReleaseInStream) ();
   STDMETHOD (Read) (void *data, UInt32 size, UInt32 *processedSize);
   HRESULT CodeResume (ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
-
+  UInt64 GetInputProcessedSize () const { return _processedIn; }
-  UInt64 GetInputProcessedSize () const { return _inSizeProcessed; }
 #endif
 
   CDecoder();

CPP/7zip/Compress/ZstdEncoder.cpp

@@ -1,48 +1,37 @@
 // ZstdEncoder.cpp
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 #include "ZstdEncoder.h"
 
-// #define SHOW_DEBUG_INFO
-#ifdef SHOW_DEBUG_INFO
-#include <stdio.h>
-#define PRF(x) x
-#else
-#define PRF(x)
-#endif
-
 #ifndef EXTRACT_ONLY
 namespace NCompress {
 namespace NZSTD {
 
 CEncoder::CEncoder():
-  _state (NULL),
+  _cstream(NULL),
-  _inBuf (NULL),
+  _buffIn(NULL),
-  _outBuf (NULL),
+  _buffOut(NULL),
-  _inPos (0),
+  _buffInSize(0),
-  _inSize (0),
+  _buffOutSize(0),
-  _inBufSizeAllocated (0),
+  _processedIn(0),
-  _outBufSizeAllocated (0),
+  _processedOut(0)
-  _inBufSize (ZBUFF_recommendedCInSize() * 30),
-  _outBufSize (ZBUFF_recommendedCOutSize() * 30),
-  _inSizeProcessed (0),
-  _outSizeProcessed (0)
 {
   _props.clear();
 }
 
 CEncoder::~CEncoder()
 {
-  if (_state)
+  if (_cstream)
-    ZBUFF_freeCCtx(_state);
+    ZSTD_freeCStream(_cstream);
 
-  MyFree (_inBuf);
+  MyFree(_buffIn);
-  MyFree (_outBuf);
+  MyFree(_buffOut);
+  _buffIn = 0;
+  _buffOut = 0;
 }
 
-STDMETHODIMP CEncoder::SetCoderProperties (const PROPID * propIDs,
+STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
-  const PROPVARIANT * coderProps, UInt32 numProps)
 {
   _props.clear();
 
@@ -57,6 +46,7 @@ STDMETHODIMP CEncoder::SetCoderProperties (const PROPID * propIDs,
         if (prop.vt != VT_UI4)
           return E_INVALIDARG;
 
+        /* level 1..22 */
         _props._level = static_cast < Byte > (prop.ulVal);
         Byte zstd_level = static_cast < Byte > (ZSTD_maxCLevel());
         if (_props._level > zstd_level)
@@ -71,6 +61,9 @@ STDMETHODIMP CEncoder::SetCoderProperties (const PROPID * propIDs,
     }
   }
 
+  _processedIn = 0;
+  _processedOut = 0;
+
   return S_OK;
 }
 
@@ -79,112 +72,77 @@ STDMETHODIMP CEncoder::WriteCoderProperties (ISequentialOutStream * outStream)
   return WriteStream(outStream, &_props, sizeof (_props));
 }
 
-HRESULT CEncoder::CreateBuffers ()
-{
-  if (_inBuf == 0 || _inBufSize != _inBufSizeAllocated)
-  {
-    MyFree (_inBuf);
-    _inBuf = (Byte *) MyAlloc (_inBufSize);
-    if (_inBuf == 0)
-      return E_OUTOFMEMORY;
-    _inBufSizeAllocated = (UInt32)_inBufSize;
-  }
-
-  if (_outBuf == 0 || _outBufSize != _outBufSizeAllocated)
-  {
-    MyFree (_outBuf);
-    _outBuf = (Byte *) MyAlloc (_outBufSize);
-    if (_outBuf == 0)
-      return E_OUTOFMEMORY;
-    _outBufSizeAllocated = (UInt32)_outBufSize;
-  }
-
-  return S_OK;
-}
-
-HRESULT CEncoder::CreateCompressor ()
-{
-  if (!_state) {
-    _state = ZBUFF_createCCtx();
-    if (!_state)
-      return S_FALSE;
-  }
-
-  if (ZBUFF_compressInit(_state, _props._level))
-    return S_FALSE;
-
-  return S_OK;
-}
-
 STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
   ISequentialOutStream *outStream, const UInt64 * /* inSize */ ,
   const UInt64 * /* outSize */ , ICompressProgressInfo *progress)
 {
-  RINOK (CreateCompressor());
+  size_t result;
-  RINOK (CreateBuffers());
 
-  UInt64 startInProgress = _inSizeProcessed;
+  /* init only once in beginning */
-  UInt64 startOutProgress = _outSizeProcessed;
+  if (!_cstream) {
 
-  for (;;)
+    /* allocate stream */
-  {
+    _cstream = ZSTD_createCStream();
-    bool eofFlag = false;
+    if (!_cstream)
-    if (_inPos == _inSize)
+      return E_OUTOFMEMORY;
-    {
+
-      _inPos = _inSize = 0;
+    /* allocate buffers */
-      RINOK (inStream->Read (_inBuf, _inBufSizeAllocated, &_inSize));
+    _buffInSize = ZSTD_CStreamInSize();
-      if (!_inSize)
+    _buffIn = MyAlloc(_buffInSize);
-	eofFlag = true;
+    if (!_buffIn)
+      return E_OUTOFMEMORY;
+
+    _buffOutSize = ZSTD_CStreamOutSize();
+    _buffOut = MyAlloc(_buffOutSize);
+    if (!_buffOut)
+      return E_OUTOFMEMORY;
   }
 
-    Byte *pIn_bytes = _inBuf + _inPos;
+  /* init or re-init stream */
-    size_t num_in_bytes = _inSize - _inPos;
+  result = ZSTD_initCStream(_cstream, _props._level);
-    Byte *pOut_bytes = _outBuf;
+  if (ZSTD_isError(result))
-    size_t num_out_bytes = _outBufSize;
-
-    size_t comp_status;
-    bool comp_finished = false;
-
-    if (eofFlag)
-    {
-      comp_status = ZBUFF_compressEnd (_state, pOut_bytes, &num_out_bytes);
-      comp_finished = (comp_status == 0);
-    }
-    else
-    {
-      comp_status = ZBUFF_compressContinue(_state, pOut_bytes, &num_out_bytes, pIn_bytes, &num_in_bytes);
-    }
-
-    bool comp_failed = ZBUFF_isError (comp_status) != 0;
-
-    if (num_in_bytes)
-    {
-      _inPos += (UInt32) num_in_bytes;
-      _inSizeProcessed += (UInt32) num_in_bytes;
-    }
-
-    if (num_out_bytes)
-    {
-      _outSizeProcessed += num_out_bytes;
-      RINOK (WriteStream (outStream, _outBuf, num_out_bytes));
-    }
-
-    if (comp_failed)
     return S_FALSE;
 
-    if (comp_finished)
+  UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
-      break;
+  for(;;) {
 
-    UInt64 inSize = _inSizeProcessed - startInProgress;
+    /* read input */
-    UInt64 outSize = _outSizeProcessed - startOutProgress;
+    RINOK(inStream->Read(_buffIn, toRead, &read));
-    if (progress)
+    size_t InSize = static_cast < size_t > (read);
-    {
+    _processedIn += InSize;
-      RINOK (progress->SetRatioInfo (&inSize, &outSize));
+
-    }
+    if (InSize == 0) {
+
+      /* @eof */
+      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+      result = ZSTD_endStream(_cstream, &output);
+      if (ZSTD_isError(result))
+        return S_FALSE;
+
+      if (output.pos) {
+        /* write last compressed bytes and update progress */
+        RINOK(WriteStream(outStream, _buffOut, output.pos));
+        _processedOut += output.pos;
+        RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
       }
 
       return S_OK;
     }
 
+    /* compress input */
+    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
+    while (input.pos < input.size) {
+      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+      result = ZSTD_compressStream(_cstream, &output , &input);
+      if (ZSTD_isError(result))
+        return S_FALSE;
+      /* write compressed stream and update progress */
+      RINOK(WriteStream(outStream, _buffOut, output.pos));
+      _processedOut += output.pos;
+      RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
+    }
+  }
+}
+
 }}
 #endif

CPP/7zip/Compress/ZstdEncoder.h

@@ -1,10 +1,9 @@
 // ZstdEncoder.h
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../../../C/Alloc.h"
 #include "../../../C/ZStd/zstd.h"
-#include "../../../C/ZStd/zbuff.h"
 
 #include "../../Common/Common.h"
 #include "../../Common/MyCom.h"
@@ -38,33 +37,22 @@ class CEncoder:
   public ICompressWriteCoderProperties,
   public CMyUnknownImp
 {
-  ZBUFF_CCtx *_state;
-
   CProps _props;
 
-  Byte *_inBuf;
+  ZSTD_CStream *_cstream;
-  Byte *_outBuf;
+  void *_buffIn;
-  UInt32 _inPos;
+  void *_buffOut;
-  UInt32 _inSize;
+  size_t _buffInSize;
-
+  size_t _buffOutSize;
-  UInt32 _inBufSizeAllocated;
+  UInt64 _processedIn;
-  UInt32 _outBufSizeAllocated;
+  UInt64 _processedOut;
-  size_t _inBufSize;
-  size_t _outBufSize;
-
-  UInt64 _inSizeProcessed;
-  UInt64 _outSizeProcessed;
 
   HRESULT CreateCompressor();
-  HRESULT CreateBuffers ();
 
 public:
   MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
-    STDMETHOD (Code) (ISequentialInStream * inStream, ISequentialOutStream *
+  STDMETHOD (Code) (ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
-      outStream, const UInt64 * inSize, const UInt64 * outSize,
+  STDMETHOD (SetCoderProperties) (const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
-      ICompressProgressInfo * progress);
-    STDMETHOD (SetCoderProperties) (const PROPID * propIDs,
-      const PROPVARIANT *props, UInt32 numProps);
   STDMETHOD (WriteCoderProperties) (ISequentialOutStream *outStream);
 
   CEncoder();

CPP/7zip/Compress/ZstdRegister.cpp

@@ -1,5 +1,5 @@
 // ZstdRegister.cpp
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 

CPP/7zip/UI/GUI/CompressDialog.cpp

@@ -1743,31 +1743,31 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory)
 
     case kZSTD:
     {
-      /* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=4194304 */
+      /* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=131075 */
       size = 0;
       switch (level) {
-        case 1: size = 1086412; decompressMemory = 677160; return size;
+        case 1: size = 824228; decompressMemory = 415024; return size;
-        case 2: size = 1283020; decompressMemory = 677160; return size;
+        case 2: size = 1282980; decompressMemory = 415024; return size;
-        case 3: size = 2855884; decompressMemory = 1201448; return size;
+        case 3: size = 922532; decompressMemory = 415024; return size;
-        case 4: size = 2102220; decompressMemory = 1201448; return size;
+        case 4: size = 1414052; decompressMemory = 415024; return size;
-        case 5: size = 2724812; decompressMemory = 1201448; return size;
+        case 5: size = 1545124; decompressMemory = 415024; return size;
-        case 6: size = 4953036; decompressMemory = 2250024; return size;
+        case 6: size = 1807268; decompressMemory = 415024; return size;
-        case 7: size = 7312332; decompressMemory = 2250024; return size;
+        case 7: size = 1807268; decompressMemory = 415024; return size;
-        case 8: size = 7836620; decompressMemory = 2250024; return size;
+        case 8: size = 1807268; decompressMemory = 415024; return size;
-        case 9: size = 10982348; decompressMemory = 2250024; return size;
+        case 9: size = 1807268; decompressMemory = 415024; return size;
-        case 10: size = 13079500; decompressMemory = 2250024; return size;
+        case 10: size = 1807268; decompressMemory = 415024; return size;
-        case 11: size = 25662412; decompressMemory = 4347176; return size;
+        case 11: size = 2331556; decompressMemory = 415024; return size;
-        case 12: size = 25662412; decompressMemory = 4347176; return size;
+        case 12: size = 2331556; decompressMemory = 415024; return size;
-        case 13: size = 29856716; decompressMemory = 4347176; return size;
+        case 13: size = 3380132; decompressMemory = 415024; return size;
-        case 14: size = 29856716; decompressMemory = 4347176; return size;
+        case 14: size = 3004832; decompressMemory = 415024; return size;
-        case 15: size = 21468108; decompressMemory = 4347176; return size;
+        case 15: size = 3004832; decompressMemory = 415024; return size;
-        case 16: size = 38245324; decompressMemory = 4347176; return size;
+        case 16: size = 4697834; decompressMemory = 415024; return size;
-        case 17: size = 55022540; decompressMemory = 4347176; return size;
+        case 17: size = 4697834; decompressMemory = 415024; return size;
-        case 18: size = 55171528; decompressMemory = 4347176; return size;
+        case 18: size = 4697834; decompressMemory = 415024; return size;
-        case 19: size = 55815954; decompressMemory = 4347176; return size;
+        case 19: size = 4697834; decompressMemory = 415024; return size;
-        case 20: size = 55815954; decompressMemory = 4347176; return size;
+        case 20: size = 4697834; decompressMemory = 415024; return size;
-        case 21: size = 55815954; decompressMemory = 4347176; return size;
+        case 21: size = 4697834; decompressMemory = 415024; return size;
-        case 22: size = 55815954; decompressMemory = 4347176; return size;
+        case 22: size = 4697834; decompressMemory = 415024; return size;
       }
       decompressMemory = 0;
       return size;

CPP/7zip/zstd.mak

@@ -8,7 +8,6 @@ ZSTD_OBJS = \
   $O\entropy_common.obj \
   $O\fse_decompress.obj \
   $O\huf_decompress.obj \
-  $O\zbuff_decompress.obj \
   $O\zstd_common.obj \
   $O\zstd_decompress.obj \
   $O\xxhash.obj \

CPP/build-x32.cmd

@@ -11,22 +11,10 @@ cd %ROOT%\Bundles\Format7z
 nmake %OPTS%
 copy O\7za.dll %OUTDIR%\7za.dll
 
-cd %ROOT%\Bundles\Format7zExtract
-nmake %OPTS%
-copy O\7zxa.dll %OUTDIR%\7zxa.dll
-
-cd %ROOT%\Bundles\Format7zExtractR
-nmake %OPTS%
-copy O\7zxr.dll %OUTDIR%\7zxr.dll
-
 cd %ROOT%\Bundles\Format7zF
 nmake %OPTS%
 copy O\7z.dll %OUTDIR%\7z.dll
 
-cd %ROOT%\Bundles\Format7zR
-nmake %OPTS%
-copy O\7zra.dll %OUTDIR%\7zra.dll
-
 cd %ROOT%\UI\FileManager
 nmake %OPTS%
 copy O\7zFM.exe %OUTDIR%\7zFM.exe
@@ -43,6 +31,10 @@ cd %ROOT%\Bundles\Codec_zstd
 nmake %OPTS%
 copy O\zstd.dll %OUTDIR%\zstd-x32.dll
 
+cd %ROOT%\Bundles\Codec_zstdF
+nmake %OPTS%
+copy O\zstd.dll %OUTDIR%\zstd-x32big.dll
+
 cd %ROOT%\..\..\C\Util\7zipInstall
 nmake %OPTS%
 copy O\7zipInstall.exe %OUTDIR%\Install-x32.exe

CPP/build-x64.cmd

@@ -11,22 +11,10 @@ cd %ROOT%\Bundles\Format7z
 nmake %OPTS%
 copy AMD64\7za.dll %OUTDIR%\7za.dll
 
-cd %ROOT%\Bundles\Format7zExtract
-nmake %OPTS%
-copy AMD64\7zxa.dll %OUTDIR%\7zxa.dll
-
-cd %ROOT%\Bundles\Format7zExtractR
-nmake %OPTS%
-copy AMD64\7zxr.dll %OUTDIR%\7zxr.dll
-
 cd %ROOT%\Bundles\Format7zF
 nmake %OPTS%
 copy AMD64\7z.dll %OUTDIR%\7z.dll
 
-cd %ROOT%\Bundles\Format7zR
-nmake %OPTS%
-copy AMD64\7zra.dll %OUTDIR%\7zra.dll
-
 cd %ROOT%\UI\FileManager
 nmake %OPTS%
 copy AMD64\7zFM.exe %OUTDIR%\7zFM.exe
@@ -43,6 +31,10 @@ cd %ROOT%\Bundles\Codec_zstd
 nmake %OPTS%
 copy AMD64\zstd.dll %OUTDIR%\zstd-x64.dll
 
+cd %ROOT%\Bundles\Codec_zstdf
+nmake %OPTS%
+copy AMD64\zstd.dll %OUTDIR%\zstd-x64big.dll
+
 cd %ROOT%\..\..\C\Util\7zipInstall
 nmake %OPTS%
 copy AMD64\7zipInstall.exe %OUTDIR%\Install-x64.exe