Switch from zbuff to stream interface

- using the recommended buffersizes from zstd lib
- compile also an codec dll with support for legacy versions

CPP/7zip/Bundles/Codec_zstd/makefile

@@ -16,7 +16,6 @@ C_OBJS = \
 ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
-  $O\zbuff_compress.obj \
   $O\zstd_compress.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Codec_zstdF/StdAfx.cpp

@@ -0,0 +1,3 @@
+// StdAfx.cpp
+
+#include "StdAfx.h"

CPP/7zip/Bundles/Codec_zstdF/StdAfx.h

@@ -0,0 +1,8 @@
+// StdAfx.h
+
+#ifndef __STDAFX_H
+#define __STDAFX_H
+
+#include "../../../Common/Common.h"
+
+#endif

CPP/7zip/Bundles/Codec_zstdF/makefile

@@ -0,0 +1,24 @@
+PROG = zstd.dll
+DEF_FILE = ../../Compress/Codec.def
+CFLAGS = $(CFLAGS) -DZSTD_LEGACY_SUPPORT
+
+7ZIP_COMMON_OBJS = \
+  $O\StreamUtils.obj \
+
+COMPRESS_OBJS = \
+  $O\CodecExports.obj \
+  $O\DllExportsCompress.obj \
+
+C_OBJS = \
+  $O\Alloc.obj \
+
+!include "../../zstd.mak"
+ZSTD_OBJS = $(ZSTD_OBJS) \
+  $O\fse_compress.obj \
+  $O\huf_compress.obj \
+  $O\zstd_compress.obj \
+  $O\zstd_v05.obj \
+  $O\zstd_v06.obj \
+  $O\zstd_v07.obj \
+
+!include "../../7zip.mak"

CPP/7zip/Bundles/Codec_zstdF/resource.rc

@@ -0,0 +1,5 @@
+#include "../../MyVersionInfo.rc"
+
+MY_VERSION_INFO_DLL("7z ZStandard Plugin", "zstd")
+
+101  ICON  "../../Archive/Icons/7z.ico"

CPP/7zip/Bundles/Format7z/makefile

@@ -1,8 +1,6 @@
 PROG = 7za.dll
 DEF_FILE = ../../Archive/Archive2.def
 CFLAGS = $(CFLAGS)
-#  -DDEFLATE_EXTRACT_ONLY \
-#  -DBZIP2_EXTRACT_ONLY \
 
 COMMON_OBJS = \
   $O\CRC.obj \
@@ -142,7 +140,6 @@ C_OBJS = \
 ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
-  $O\zbuff_compress.obj \
   $O\zstd_compress.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zF/makefile

@@ -1,7 +1,7 @@
 PROG = 7z.dll
 DEF_FILE = ../../Archive/Archive2.def
 CFLAGS = $(CFLAGS) \
-  -DEXTERNAL_CODECS -DZSTD_LEGACY \
+  -DEXTERNAL_CODECS -DZSTD_LEGACY_SUPPORT \
 
 !IFNDEF UNDER_CE
 CFLAGS = $(CFLAGS) -D_7ZIP_LARGE_PAGES
@@ -21,10 +21,9 @@ AR_OBJS = $(AR_OBJS) \
 ZSTD_OBJS = $(ZSTD_OBJS) \
   $O\fse_compress.obj \
   $O\huf_compress.obj \
-  $O\zbuff_compress.obj \
   $O\zstd_compress.obj \
-  $O\zstd_v07.obj \
-  $O\zstd_v06.obj \
   $O\zstd_v05.obj \
+  $O\zstd_v06.obj \
+  $O\zstd_v07.obj \
 
 !include "../../7zip.mak"

CPP/7zip/Compress/ZstdDecoder.cpp

@@ -1,418 +1,263 @@
 // ZstdDecoder.cpp
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 #include "ZstdDecoder.h"
 
-// #define SHOW_DEBUG_INFO
-#ifdef SHOW_DEBUG_INFO
-#include <stdio.h>
-#define PRF(x) x
-#else
-#define PRF(x)
-#endif
-
 namespace NCompress {
 namespace NZSTD {
 
-CDecoder::CDecoder ():
-  _inBuf (NULL),
-  _outBuf (NULL),
-  _inPos (0),
-  _inSize (0),
-  _eofFlag (false),
-  _state (NULL),
-  _propsWereSet (false),
-  _outSizeDefined (false),
-  _outSize (0),
-
-  _inSizeProcessed (0),
-  _outSizeProcessed (0),
-  _inBufSizeAllocated (0),
-  _outBufSizeAllocated (0),
-
-  _inBufSize (ZBUFF_recommendedDInSize() * 30),
-  _outBufSize (ZBUFF_recommendedDOutSize()* 30)
+CDecoder::CDecoder():
+  _dstream(NULL),
+  _buffIn(NULL),
+  _buffOut(NULL),
+  _buffInSizeAllocated(0),
+  _buffOutSizeAllocated(0),
+  _buffInSize(ZSTD_DStreamInSize()),
+  _buffOutSize(ZSTD_DStreamOutSize()*4),
+  _processedIn(0),
+  _processedOut(0),
+  _propsWereSet(false)
 {
-  _props.clear ();
+  _props.clear();
 }
 
-CDecoder::~CDecoder ()
+CDecoder::~CDecoder()
 {
-  if (_state)
-    ZB_freeDCtx(_state);
+  if (_dstream)
+    ZSTD_freeDStream(_dstream);
 
-  MyFree (_inBuf);
-  MyFree (_outBuf);
+  MyFree(_buffIn);
+  MyFree(_buffOut);
+
+  _buffInSizeAllocated = 0;
+  _buffOutSizeAllocated = 0;
 }
 
-STDMETHODIMP CDecoder::SetInBufSize (UInt32, UInt32 size)
+STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size)
 {
-  _inBufSize = size;
-  return S_OK;
-}
+  DProps *pProps = (DProps *)prop;
 
-STDMETHODIMP CDecoder::SetOutBufSize (UInt32, UInt32 size)
-{
-  _outBufSize = size;
-  return S_OK;
-}
-
-HRESULT CDecoder::CreateBuffers ()
-{
-  if (_inBuf == 0 || _inBufSize != _inBufSizeAllocated)
-  {
-    MyFree (_inBuf);
-    _inBuf = (Byte *) MyAlloc (_inBufSize);
-    if (_inBuf == 0)
-      return E_OUTOFMEMORY;
-    _inBufSizeAllocated = (UInt32)_inBufSize;
-  }
-
-  if (_outBuf == 0 || _outBufSize != _outBufSizeAllocated)
-  {
-    MyFree (_outBuf);
-    _outBuf = (Byte *) MyAlloc (_outBufSize);
-    if (_outBuf == 0)
-      return E_OUTOFMEMORY;
-    _outBufSizeAllocated = (UInt32)_outBufSize;
-  }
-
-  return S_OK;
-}
-
-STDMETHODIMP CDecoder::SetDecoderProperties2 (const Byte * prop, UInt32 size)
-{
-  DProps *pProps = (DProps *) prop;
-
-  if (size != sizeof (DProps))
+  if (size != sizeof(DProps))
     return E_FAIL;
 
-  // version 0.x currently
+#ifdef ZSTD_LEGACY_SUPPORT
+  /* version 0.x and 1.x are okay */
+  if (pProps->_ver_major > 1)
+    return E_FAIL;
+
+  /* 0.5, 0.6, 0.7, 0.8 are supported! */
+  if (pProps->_ver_major == 0) {
+    switch (pProps->_ver_minor) {
+    case 5:
+      break;
+    case 6:
+      break;
+    case 7:
+      break;
+    case 8:
+      break;
+    default:
+      return E_FAIL;
+  }}
+#else
+  /* only exact version is okay */
   if (pProps->_ver_major != ZSTD_VERSION_MAJOR)
     return E_FAIL;
-
-  switch (pProps->_ver_minor) {
-  case ZSTD_VERSION_MINOR:
-    break;
-#ifdef ZSTD_LEGACY
-  case 5:
-    break;
-  case 6:
-    break;
-  case 7:
-    break;
-#endif
-  default:
+  if (pProps->_ver_minor != ZSTD_VERSION_MINOR)
     return E_FAIL;
-  }
+
+#endif
 
   memcpy(&_props, pProps, sizeof (DProps));
   _propsWereSet = true;
 
-  return CreateBuffers();
+  return S_OK;
 }
 
 HRESULT CDecoder::CreateDecompressor()
 {
+  size_t result;
+
   if (!_propsWereSet)
     return E_FAIL;
 
-  if (!_state) {
-    _state = ZB_createDCtx();
-    if (!_state)
+  if (!_dstream) {
+    _dstream = ZSTD_createDStream();
+    if (!_dstream)
       return E_FAIL;
   }
 
-  if (ZBUFF_isError(ZB_decompressInit(_state)))
+  result = ZSTD_initDStream(_dstream);
+  if (ZSTD_isError(result))
     return E_FAIL;
 
-  _eofFlag = false;
-
-  return S_OK;
-}
-
-HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * outSize)
-{
-  _outSizeDefined = (outSize != NULL);
-  if (_outSizeDefined)
-    _outSize = *outSize;
-  _outSizeProcessed = 0;
-
-  RINOK (CreateDecompressor());
-
-  return S_OK;
-}
-
-STDMETHODIMP CDecoder::SetOutStreamSize (const UInt64 * outSize)
-{
-  _inSizeProcessed = 0;
-  _inPos = _inSize = 0;
-  RINOK (SetOutStreamSizeResume (outSize));
-  return S_OK;
-}
-
-HRESULT CDecoder::CodeSpec (ISequentialInStream * inStream,
-		      ISequentialOutStream * outStream,
-		      ICompressProgressInfo * progress)
-{
-  if (_inBuf == 0 || !_propsWereSet)
-    return S_FALSE;
-
-  if (!_state)
+  /* allocate buffers */
+  if (_buffInSizeAllocated != _buffInSize)
   {
-    if (CreateDecompressor () != S_OK)
-      return E_FAIL;
+    if (_buffIn)
+      MyFree(_buffIn);
+    _buffIn = MyAlloc(_buffInSize);
+
+    if (!_buffIn)
+      return E_OUTOFMEMORY;
+    _buffInSizeAllocated = _buffInSize;
   }
 
-  UInt64 startInProgress = _inSizeProcessed;
-
-  for (;;)
+  if (_buffOutSizeAllocated != _buffOutSize)
   {
-    if ((!_eofFlag) && (_inPos == _inSize))
-    {
-      _inPos = _inSize = 0;
-      RINOK (inStream->Read (_inBuf, _inBufSizeAllocated, &_inSize));
-      if (!_inSize)
-	_eofFlag = true;
-    }
+    if (_buffOut)
+      MyFree(_buffOut);
+    _buffOut = MyAlloc(_buffOutSize);
 
-    Byte *pIn_bytes = _inBuf + _inPos;
-    size_t num_in_bytes = _inSize - _inPos;
-    Byte *pOut_bytes = _outBuf;
-    size_t num_out_bytes = _outBufSize;
-    if (_outSizeDefined)
-    {
-      UInt64 out_remaining = _outSize - _outSizeProcessed;
-      if (out_remaining == 0)
-	return S_OK;
-      if (num_out_bytes > out_remaining)
-	num_out_bytes = static_cast < size_t > (out_remaining);
-    }
-
-    size_t decomp_status =
-      ZB_decompressContinue (_state, pOut_bytes, &num_out_bytes,
-				pIn_bytes, &num_in_bytes);
-    bool decomp_finished = (decomp_status == 0);
-    bool decomp_failed = ZBUFF_isError (decomp_status) != 0;
-
-    if (num_in_bytes)
-    {
-      _inPos += (UInt32) num_in_bytes;
-      _inSizeProcessed += (UInt32) num_in_bytes;
-    }
-
-    if (num_out_bytes)
-    {
-      _outSizeProcessed += num_out_bytes;
-
-      RINOK (WriteStream (outStream, _outBuf, num_out_bytes));
-    }
-
-    if (decomp_failed)
-    {
-      PRF(fprintf(stderr, "zstdcodec: ZB_decompressContinue() failed: %s\n", ZBUFF_getErrorName(decomp_status)));
-      return S_FALSE;
-    }
-
-    if (decomp_finished)
-      break;
-
-    // This check is to prevent locking up if the input file is accidently truncated.
-    bool made_forward_progress = (num_out_bytes != 0) || (num_in_bytes != 0);
-    if ((!made_forward_progress) && (_eofFlag))
-    {
-      return S_FALSE;
-    }
-
-    UInt64 inSize = _inSizeProcessed - startInProgress;
-    if (progress)
-    {
-      RINOK (progress->SetRatioInfo (&inSize, &_outSizeProcessed));
-    }
+    if (!_buffOut)
+      return E_OUTOFMEMORY;
+    _buffOutSizeAllocated = _buffOutSize;
   }
 
   return S_OK;
 }
 
-STDMETHODIMP CDecoder::Code (ISequentialInStream * inStream,
-		  ISequentialOutStream * outStream,
-		  const UInt64 * inSize,
-		  const UInt64 * outSize, ICompressProgressInfo * progress)
+HRESULT CDecoder::SetOutStreamSizeResume(const UInt64 * /*outSize*/)
 {
-  (void) inSize;
-  if (_inBuf == 0)
-    return E_INVALIDARG;
-  SetOutStreamSize (outSize);
-  return CodeSpec (inStream, outStream, progress);
+  _processedOut = 0;
+  RINOK(CreateDecompressor());
+
+  return S_OK;
 }
 
-// wrapper for different versions
-void *CDecoder::ZB_createDCtx(void)
+STDMETHODIMP CDecoder::SetOutStreamSize(const UInt64 * outSize)
 {
-  PRF(fprintf(stderr, "zstdcodec: ZB_createDCtx(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return (void*)ZBUFFv05_createDCtx();
-    break;
-  case 6:
-    return (void*)ZBUFFv06_createDCtx();
-    break;
-  case 7:
-    return (void*)ZBUFFv07_createDCtx();
-    break;
-  }
-#endif
-  return (void*)ZBUFF_createDCtx();
+  _processedIn = 0;
+  RINOK(SetOutStreamSizeResume(outSize));
+
+  return S_OK;
 }
 
-size_t CDecoder::ZB_freeDCtx(void *dctx)
+STDMETHODIMP CDecoder::SetInBufSize(UInt32, UInt32 size)
 {
-  PRF(fprintf(stderr, "zstdcodec: ZB_freeDCtx(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return ZBUFFv05_freeDCtx((ZBUFFv05_DCtx *)dctx);
-    break;
-  case 6:
-    return ZBUFFv06_freeDCtx((ZBUFFv06_DCtx *)dctx);
-    break;
-  case 7:
-    return ZBUFFv07_freeDCtx((ZBUFFv07_DCtx *)dctx);
-    break;
-  }
-#endif
-  return ZBUFF_freeDCtx((ZBUFF_DCtx *)dctx);
+  _buffInSize = size;
+  return S_OK;
 }
 
-size_t CDecoder::ZB_decompressInit(void *dctx)
+STDMETHODIMP CDecoder::SetOutBufSize(UInt32, UInt32 size)
 {
-  PRF(fprintf(stderr, "zstdcodec: ZB_decompressInit(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return ZBUFFv05_decompressInit((ZBUFFv05_DCtx *)dctx);
-    break;
-  case 6:
-    return ZBUFFv06_decompressInit((ZBUFFv06_DCtx *)dctx);
-    break;
-  case 7:
-    return ZBUFFv07_decompressInit((ZBUFFv07_DCtx *)dctx);
-    break;
-  }
-#endif
-  return ZBUFF_decompressInit((ZBUFF_DCtx *)dctx);
+  _buffOutSize = size;
+  return S_OK;
 }
 
-size_t CDecoder::ZB_decompressContinue(void *dctx, void* dst, size_t *dstCapacityPtr, const void* src, size_t *srcSizePtr)
+HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress)
 {
-  PRF(fprintf(stderr, "zstdcodec: ZB_decompressContinue(v=%d)\n", _props._ver_minor));
-#ifdef ZSTD_LEGACY
-  switch (_props._ver_minor) {
-  case 5:
-    return ZBUFFv05_decompressContinue((ZBUFFv05_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-    break;
-  case 6:
-    return ZBUFFv06_decompressContinue((ZBUFFv06_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-    break;
-  case 7:
-    return ZBUFFv07_decompressContinue((ZBUFFv07_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
-    break;
+  RINOK(CreateDecompressor());
+
+  size_t result;
+  UInt32 const toRead = static_cast < const UInt32 > (_buffInSize);
+  for(;;) {
+    UInt32 read;
+
+    /* read input */
+    RINOK(inStream->Read(_buffIn, toRead, &read));
+    size_t InSize = static_cast < size_t > (read);
+    _processedIn += InSize;
+
+    if (InSize == 0)
+      return S_OK;
+
+    /* decompress input */
+    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
+    for (;;) {
+      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+      result = ZSTD_decompressStream(_dstream, &output , &input);
+      if (ZSTD_isError(result))
+        return S_FALSE;
+      /* write decompressed stream and update progress */
+      RINOK(WriteStream(outStream, _buffOut, output.pos));
+      _processedOut += output.pos;
+      RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
+
+      /* one more round */
+      if ((input.pos == input.size) && (result == 1))
+        continue;
+
+      /* finished */
+      if (input.pos == input.size)
+        break;
+    }
   }
-#endif
-  return ZBUFF_decompressContinue((ZBUFF_DCtx *)dctx, dst, dstCapacityPtr, src, srcSizePtr);
+}
+
+STDMETHODIMP CDecoder::Code(ISequentialInStream * inStream, ISequentialOutStream * outStream,
+  const UInt64 * /*inSize */, const UInt64 *outSize, ICompressProgressInfo * progress)
+{
+  SetOutStreamSize(outSize);
+  return CodeSpec(inStream, outStream, progress);
 }
 
 #ifndef NO_READ_FROM_CODER
-STDMETHODIMP CDecoder::SetInStream (ISequentialInStream * inStream)
+STDMETHODIMP CDecoder::SetInStream(ISequentialInStream * inStream)
 {
   _inStream = inStream;
   return S_OK;
 }
 
-STDMETHODIMP CDecoder::ReleaseInStream ()
+STDMETHODIMP CDecoder::ReleaseInStream()
 {
-  _inStream.Release ();
+  _inStream.Release();
+
   return S_OK;
 }
 
-STDMETHODIMP CDecoder::Read (void *data, UInt32 size, UInt32 * processedSize)
+STDMETHODIMP CDecoder::Read(void *data, UInt32 /*size*/, UInt32 *processedSize)
 {
   if (processedSize)
     *processedSize = 0;
 
-  if (_inBuf == 0 || !_propsWereSet)
-    return S_FALSE;
+  size_t result;
 
-  if (!_state)
-  {
-    if (CreateDecompressor () != S_OK)
+  if (!_dstream)
+    if (CreateDecompressor() != S_OK)
       return E_FAIL;
-  }
 
-  while (size != 0)
-  {
-    if ((!_eofFlag) && (_inPos == _inSize))
-    {
-      _inPos = _inSize = 0;
-      RINOK (_inStream->Read (_inBuf, _inBufSizeAllocated, &_inSize));
-      if (!_inSize)
-	_eofFlag = true;
+  UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
+  Byte *dataout = static_cast < Byte* > (data);
+  for(;;) {
+    /* read input */
+    RINOK(_inStream->Read(_buffIn, toRead, &read));
+    size_t InSize = static_cast < size_t > (read);
+    _processedIn += InSize;
+
+    if (InSize == 0) {
+      return S_OK;
     }
 
-    Byte *pIn_bytes = _inBuf + _inPos;
-    size_t  num_in_bytes = _inSize - _inPos;
-    Byte *pOut_bytes = (Byte *) data;
-    size_t  num_out_bytes = size;
+    /* decompress input */
+    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
+    for (;;) {
+      ZSTD_outBuffer output = { dataout, _buffOutSize, 0 };
+      result = ZSTD_decompressStream(_dstream, &output , &input);
+      if (ZSTD_isError(result))
+        return S_FALSE;
 
-    size_t decomp_status =
-      ZB_decompressContinue (_state, pOut_bytes, &num_out_bytes,
-				pIn_bytes, &num_in_bytes);
-    bool
-      decomp_finished = (decomp_status == 0);
-    bool
-      decomp_failed = ZBUFF_isError (decomp_status) != 0;
-
-    if (num_in_bytes)
-    {
-      _inPos += (UInt32) num_in_bytes;
-      _inSizeProcessed += num_in_bytes;
-    }
-
-    if (num_out_bytes)
-    {
-      _outSizeProcessed += num_out_bytes;
-      size -= (UInt32) num_out_bytes;
       if (processedSize)
-	*processedSize += (UInt32) num_out_bytes;
-    }
+        *processedSize += static_cast < UInt32 > (output.pos);
 
-    if (decomp_failed)
-    {
-      return S_FALSE;
-    }
+      dataout += output.pos;
 
-    if (decomp_finished)
-      break;
+      /* one more round */
+      if ((input.pos == input.size) && (result == 1))
+        continue;
 
-    // This check is to prevent locking up if the input file is accidently truncated.
-    bool made_forward_progress = (num_out_bytes != 0) || (num_in_bytes != 0);
-    if ((!made_forward_progress) && (_eofFlag))
-    {
-      return S_FALSE;
+      /* finished */
+      if (input.pos == input.size)
+        break;
     }
   }
-
-  return S_OK;
 }
 
-HRESULT CDecoder::CodeResume (ISequentialOutStream * outStream,
-			const UInt64 * outSize,
-			ICompressProgressInfo * progress)
+HRESULT CDecoder::CodeResume(ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress)
 {
-  RINOK (SetOutStreamSizeResume (outSize));
-  return CodeSpec (_inStream, outStream, progress);
+  RINOK(SetOutStreamSizeResume(outSize));
+  return CodeSpec(_inStream, outStream, progress);
 }
 #endif
 

CPP/7zip/Compress/ZstdDecoder.h

@@ -1,11 +1,16 @@
 // ZstdDecoder.h
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
+
+/**
+ * you can define ZSTD_LEGACY_SUPPORT to be backwards compatible
+ * with these versions: 0.5, 0.6, 0.7, 0.8 (0.8 == 1.0)
+ *
+ * /TR 2016-09-04
+ */
 
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../../../C/Alloc.h"
 #include "../../../C/ZStd/zstd.h"
-#include "../../../C/ZStd/zbuff.h"
-#include "../../../C/ZStd/zstd_legacy.h"
 
 #include "../../Common/Common.h"
 #include "../../Common/MyCom.h"
@@ -21,7 +26,7 @@ struct DProps
   DProps() { clear (); }
   void clear ()
   {
-    memset (this, 0, sizeof (*this));
+    memset(this, 0, sizeof (*this));
     _ver_major = ZSTD_VERSION_MAJOR;
     _ver_minor = ZSTD_VERSION_MINOR;
     _level = 1;
@@ -42,37 +47,25 @@ class CDecoder:public ICompressCoder,
   public CMyUnknownImp
 {
   CMyComPtr < ISequentialInStream > _inStream;
-  Byte *_inBuf;
-  Byte *_outBuf;
-  UInt32 _inPos;
-  UInt32 _inSize;
-  bool _eofFlag;
-
-  void *_state;
 
   DProps _props;
   bool _propsWereSet;
 
-  bool _outSizeDefined;
-  UInt64 _outSize;
-  UInt64 _inSizeProcessed;
-  UInt64 _outSizeProcessed;
+  ZSTD_DStream *_dstream;
+  void *_buffIn;
+  void *_buffOut;
 
-  UInt32 _inBufSizeAllocated;
-  UInt32 _outBufSizeAllocated;
-  size_t _inBufSize;
-  size_t _outBufSize;
+  size_t _buffInSize;
+  size_t _buffOutSize;
+  size_t _buffInSizeAllocated;
+  size_t _buffOutSizeAllocated;
 
-  HRESULT CreateBuffers ();
-  HRESULT CodeSpec (ISequentialInStream * inStream, ISequentialOutStream * outStream, ICompressProgressInfo * progress);
-  HRESULT SetOutStreamSizeResume (const UInt64 * outSize);
-  HRESULT CreateDecompressor ();
+  UInt64 _processedIn;
+  UInt64 _processedOut;
 
-  // wrapper for different versions
-  void *ZB_createDCtx(void);
-  size_t ZB_freeDCtx(void *dctx);
-  size_t ZB_decompressInit(void *dctx);
-  size_t ZB_decompressContinue(void *dctx, void* dst, size_t *dstCapacityPtr, const void* src, size_t *srcSizePtr);
+  HRESULT CDecoder::CreateDecompressor();
+  HRESULT CodeSpec(ISequentialInStream *inStream, ISequentialOutStream *outStream, ICompressProgressInfo *progress);
+  HRESULT SetOutStreamSizeResume(const UInt64 *outSize);
 
 public:
 
@@ -87,19 +80,18 @@ public:
   MY_QUERYINTERFACE_END
 
   MY_ADDREF_RELEASE
-  STDMETHOD (Code)(ISequentialInStream * inStream, ISequentialOutStream * outStream, const UInt64 * inSize, const UInt64 * outSize, ICompressProgressInfo * progress);
-  STDMETHOD (SetDecoderProperties2) (const Byte * data, UInt32 size);
-  STDMETHOD (SetOutStreamSize) (const UInt64 * outSize);
+  STDMETHOD (Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (SetDecoderProperties2) (const Byte *data, UInt32 size);
+  STDMETHOD (SetOutStreamSize) (const UInt64 *outSize);
   STDMETHOD (SetInBufSize) (UInt32 streamIndex, UInt32 size);
   STDMETHOD (SetOutBufSize) (UInt32 streamIndex, UInt32 size);
 
 #ifndef NO_READ_FROM_CODER
-  STDMETHOD (SetInStream) (ISequentialInStream * inStream);
+  STDMETHOD (SetInStream) (ISequentialInStream *inStream);
   STDMETHOD (ReleaseInStream) ();
-  STDMETHOD (Read) (void *data, UInt32 size, UInt32 * processedSize);
-  HRESULT CodeResume (ISequentialOutStream * outStream, const UInt64 * outSize, ICompressProgressInfo * progress);
-
-  UInt64 GetInputProcessedSize () const { return _inSizeProcessed; }
+  STDMETHOD (Read) (void *data, UInt32 size, UInt32 *processedSize);
+  HRESULT CodeResume (ISequentialOutStream *outStream, const UInt64 *outSize, ICompressProgressInfo *progress);
+  UInt64 GetInputProcessedSize () const { return _processedIn; }
 #endif
 
   CDecoder();

CPP/7zip/Compress/ZstdEncoder.cpp

@@ -1,50 +1,39 @@
 // ZstdEncoder.cpp
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 #include "ZstdEncoder.h"
 
-// #define SHOW_DEBUG_INFO
-#ifdef SHOW_DEBUG_INFO
-#include <stdio.h>
-#define PRF(x) x
-#else
-#define PRF(x)
-#endif
-
 #ifndef EXTRACT_ONLY
 namespace NCompress {
 namespace NZSTD {
 
 CEncoder::CEncoder():
-  _state (NULL),
-  _inBuf (NULL),
-  _outBuf (NULL),
-  _inPos (0),
-  _inSize (0),
-  _inBufSizeAllocated (0),
-  _outBufSizeAllocated (0),
-  _inBufSize (ZBUFF_recommendedCInSize() * 30),
-  _outBufSize (ZBUFF_recommendedCOutSize() * 30),
-  _inSizeProcessed (0),
-  _outSizeProcessed (0)
+  _cstream(NULL),
+  _buffIn(NULL),
+  _buffOut(NULL),
+  _buffInSize(0),
+  _buffOutSize(0),
+  _processedIn(0),
+  _processedOut(0)
 {
-  _props.clear ();
+  _props.clear();
 }
 
-CEncoder::~CEncoder ()
+CEncoder::~CEncoder()
 {
-  if (_state)
-    ZBUFF_freeCCtx(_state);
+  if (_cstream)
+    ZSTD_freeCStream(_cstream);
 
-  MyFree (_inBuf);
-  MyFree (_outBuf);
+  MyFree(_buffIn);
+  MyFree(_buffOut);
+  _buffIn = 0;
+  _buffOut = 0;
 }
 
-STDMETHODIMP CEncoder::SetCoderProperties (const PROPID * propIDs,
-  const PROPVARIANT * coderProps, UInt32 numProps)
+STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARIANT * coderProps, UInt32 numProps)
 {
-  _props.clear ();
+  _props.clear();
 
   for (UInt32 i = 0; i < numProps; i++)
   {
@@ -54,136 +43,105 @@ STDMETHODIMP CEncoder::SetCoderProperties (const PROPID * propIDs,
     {
     case NCoderPropID::kLevel:
       {
-	if (prop.vt != VT_UI4)
-	  return E_INVALIDARG;
+        if (prop.vt != VT_UI4)
+          return E_INVALIDARG;
 
-	_props._level = static_cast < Byte > (prop.ulVal);
-	Byte zstd_level = static_cast < Byte > (ZSTD_maxCLevel());
-	if (_props._level > zstd_level)
-	  _props._level = zstd_level;
+        /* level 1..22 */
+        _props._level = static_cast < Byte > (prop.ulVal);
+        Byte zstd_level = static_cast < Byte > (ZSTD_maxCLevel());
+        if (_props._level > zstd_level)
+          _props._level = zstd_level;
 
-	break;
+        break;
       }
     default:
       {
-	break;
+        break;
       }
     }
   }
 
-  return S_OK;
-}
-
-STDMETHODIMP CEncoder::WriteCoderProperties (ISequentialOutStream * outStream)
-{
-  return WriteStream (outStream, &_props, sizeof (_props));
-}
-
-HRESULT CEncoder::CreateBuffers ()
-{
-  if (_inBuf == 0 || _inBufSize != _inBufSizeAllocated)
-  {
-    MyFree (_inBuf);
-    _inBuf = (Byte *) MyAlloc (_inBufSize);
-    if (_inBuf == 0)
-      return E_OUTOFMEMORY;
-    _inBufSizeAllocated = (UInt32)_inBufSize;
-  }
-
-  if (_outBuf == 0 || _outBufSize != _outBufSizeAllocated)
-  {
-    MyFree (_outBuf);
-    _outBuf = (Byte *) MyAlloc (_outBufSize);
-    if (_outBuf == 0)
-      return E_OUTOFMEMORY;
-    _outBufSizeAllocated = (UInt32)_outBufSize;
-  }
+  _processedIn = 0;
+  _processedOut = 0;
 
   return S_OK;
 }
 
-HRESULT CEncoder::CreateCompressor ()
+STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream * outStream)
 {
-  if (!_state) {
-    _state = ZBUFF_createCCtx();
-    if (!_state)
-      return S_FALSE;
+  return WriteStream(outStream, &_props, sizeof (_props));
+}
+
+STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
+  ISequentialOutStream *outStream, const UInt64 * /* inSize */ ,
+  const UInt64 * /* outSize */ , ICompressProgressInfo *progress)
+{
+  size_t result;
+
+  /* init only once in beginning */
+  if (!_cstream) {
+
+    /* allocate stream */
+    _cstream = ZSTD_createCStream();
+    if (!_cstream)
+      return E_OUTOFMEMORY;
+
+    /* allocate buffers */
+    _buffInSize = ZSTD_CStreamInSize();
+    _buffIn = MyAlloc(_buffInSize);
+    if (!_buffIn)
+      return E_OUTOFMEMORY;
+
+    _buffOutSize = ZSTD_CStreamOutSize();
+    _buffOut = MyAlloc(_buffOutSize);
+    if (!_buffOut)
+      return E_OUTOFMEMORY;
   }
 
-  if (ZBUFF_compressInit(_state, _props._level))
+  /* init or re-init stream */
+  result = ZSTD_initCStream(_cstream, _props._level);
+  if (ZSTD_isError(result))
     return S_FALSE;
 
-  return S_OK;
-}
+  UInt32 read, toRead = static_cast < UInt32 > (_buffInSize);
+  for(;;) {
 
-STDMETHODIMP CEncoder::Code (ISequentialInStream * inStream,
-  ISequentialOutStream * outStream, const UInt64 * /* inSize */ ,
-  const UInt64 * /* outSize */ , ICompressProgressInfo * progress)
-{
-  RINOK (CreateCompressor());
-  RINOK (CreateBuffers());
+    /* read input */
+    RINOK(inStream->Read(_buffIn, toRead, &read));
+    size_t InSize = static_cast < size_t > (read);
+    _processedIn += InSize;
 
-  UInt64 startInProgress = _inSizeProcessed;
-  UInt64 startOutProgress = _outSizeProcessed;
+    if (InSize == 0) {
 
-  for (;;)
-  {
-    bool eofFlag = false;
-    if (_inPos == _inSize)
-    {
-      _inPos = _inSize = 0;
-      RINOK (inStream->Read (_inBuf, _inBufSizeAllocated, &_inSize));
-      if (!_inSize)
-	eofFlag = true;
+      /* @eof */
+      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+      result = ZSTD_endStream(_cstream, &output);
+      if (ZSTD_isError(result))
+        return S_FALSE;
+
+      if (output.pos) {
+        /* write last compressed bytes and update progress */
+        RINOK(WriteStream(outStream, _buffOut, output.pos));
+        _processedOut += output.pos;
+        RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
+      }
+
+      return S_OK;
     }
 
-    Byte *pIn_bytes = _inBuf + _inPos;
-    size_t num_in_bytes = _inSize - _inPos;
-    Byte *pOut_bytes = _outBuf;
-    size_t num_out_bytes = _outBufSize;
-
-    size_t comp_status;
-    bool comp_finished = false;
-
-    if (eofFlag)
-    {
-      comp_status = ZBUFF_compressEnd (_state, pOut_bytes, &num_out_bytes);
-      comp_finished = (comp_status == 0);
-    }
-    else
-    {
-      comp_status = ZBUFF_compressContinue(_state, pOut_bytes, &num_out_bytes, pIn_bytes, &num_in_bytes);
-    }
-
-    bool comp_failed = ZBUFF_isError (comp_status) != 0;
-
-    if (num_in_bytes)
-    {
-      _inPos += (UInt32) num_in_bytes;
-      _inSizeProcessed += (UInt32) num_in_bytes;
-    }
-
-    if (num_out_bytes)
-    {
-      _outSizeProcessed += num_out_bytes;
-      RINOK (WriteStream (outStream, _outBuf, num_out_bytes));
-    }
-
-    if (comp_failed)
-      return S_FALSE;
-
-    if (comp_finished)
-      break;
-
-    UInt64 inSize = _inSizeProcessed - startInProgress;
-    UInt64 outSize = _outSizeProcessed - startOutProgress;
-    if (progress)
-    {
-      RINOK (progress->SetRatioInfo (&inSize, &outSize));
+    /* compress input */
+    ZSTD_inBuffer input = { _buffIn, InSize, 0 };
+    while (input.pos < input.size) {
+      ZSTD_outBuffer output = { _buffOut, _buffOutSize, 0 };
+      result = ZSTD_compressStream(_cstream, &output , &input);
+      if (ZSTD_isError(result))
+        return S_FALSE;
+      /* write compressed stream and update progress */
+      RINOK(WriteStream(outStream, _buffOut, output.pos));
+      _processedOut += output.pos;
+      RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut));
     }
   }
-
-  return S_OK;
 }
 
 }}

CPP/7zip/Compress/ZstdEncoder.h

@@ -1,10 +1,9 @@
 // ZstdEncoder.h
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #define ZSTD_STATIC_LINKING_ONLY
 #include "../../../C/Alloc.h"
 #include "../../../C/ZStd/zstd.h"
-#include "../../../C/ZStd/zbuff.h"
 
 #include "../../Common/Common.h"
 #include "../../Common/MyCom.h"
@@ -20,7 +19,7 @@ struct CProps
   CProps() { clear (); }
   void clear ()
   {
-    memset (this, 0, sizeof (*this));
+    memset(this, 0, sizeof (*this));
     _ver_major = ZSTD_VERSION_MAJOR;
     _ver_minor = ZSTD_VERSION_MINOR;
     _level = 3;
@@ -38,37 +37,26 @@ class CEncoder:
   public ICompressWriteCoderProperties,
   public CMyUnknownImp
 {
-  ZBUFF_CCtx *_state;
-
   CProps _props;
 
-  Byte *_inBuf;
-  Byte *_outBuf;
-  UInt32 _inPos;
-  UInt32 _inSize;
+  ZSTD_CStream *_cstream;
+  void *_buffIn;
+  void *_buffOut;
+  size_t _buffInSize;
+  size_t _buffOutSize;
+  UInt64 _processedIn;
+  UInt64 _processedOut;
 
-  UInt32 _inBufSizeAllocated;
-  UInt32 _outBufSizeAllocated;
-  size_t _inBufSize;
-  size_t _outBufSize;
-
-  UInt64 _inSizeProcessed;
-  UInt64 _outSizeProcessed;
-
-  HRESULT CreateCompressor ();
-  HRESULT CreateBuffers ();
+  HRESULT CreateCompressor();
 
 public:
-    MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
-    STDMETHOD (Code) (ISequentialInStream * inStream, ISequentialOutStream *
-      outStream, const UInt64 * inSize, const UInt64 * outSize,
-      ICompressProgressInfo * progress);
-    STDMETHOD (SetCoderProperties) (const PROPID * propIDs,
-      const PROPVARIANT *props, UInt32 numProps);
-    STDMETHOD (WriteCoderProperties) (ISequentialOutStream * outStream);
+  MY_UNKNOWN_IMP2 (ICompressSetCoderProperties, ICompressWriteCoderProperties)
+  STDMETHOD (Code) (ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD (SetCoderProperties) (const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
+  STDMETHOD (WriteCoderProperties) (ISequentialOutStream *outStream);
 
-    CEncoder();
-    virtual ~CEncoder();
+  CEncoder();
+  virtual ~CEncoder();
 };
 
 }}

CPP/7zip/Compress/ZstdRegister.cpp

@@ -1,5 +1,5 @@
 // ZstdRegister.cpp
-// (C) 2016 Rich Geldreich, Tino Reichardt
+// (C) 2016 Tino Reichardt
 
 #include "StdAfx.h"
 

CPP/7zip/UI/GUI/CompressDialog.cpp

@@ -1740,34 +1740,34 @@ UInt64 CCompressDialog::GetMemoryUsage(UInt32 dict, UInt64 &decompressMemory)
       decompressMemory = dict + (2 << 20);
       return size + decompressMemory;
     }
-    
+
     case kZSTD:
     {
-      /* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=4194304 */
+      /* Code Snippet for CPP/7zip/UI/GUI/CompressDialog.cpp with blocklen=131075 */
       size = 0;
       switch (level) {
-        case 1: size = 1086412; decompressMemory = 677160; return size;
-        case 2: size = 1283020; decompressMemory = 677160; return size;
-        case 3: size = 2855884; decompressMemory = 1201448; return size;
-        case 4: size = 2102220; decompressMemory = 1201448; return size;
-        case 5: size = 2724812; decompressMemory = 1201448; return size;
-        case 6: size = 4953036; decompressMemory = 2250024; return size;
-        case 7: size = 7312332; decompressMemory = 2250024; return size;
-        case 8: size = 7836620; decompressMemory = 2250024; return size;
-        case 9: size = 10982348; decompressMemory = 2250024; return size;
-        case 10: size = 13079500; decompressMemory = 2250024; return size;
-        case 11: size = 25662412; decompressMemory = 4347176; return size;
-        case 12: size = 25662412; decompressMemory = 4347176; return size;
-        case 13: size = 29856716; decompressMemory = 4347176; return size;
-        case 14: size = 29856716; decompressMemory = 4347176; return size;
-        case 15: size = 21468108; decompressMemory = 4347176; return size;
-        case 16: size = 38245324; decompressMemory = 4347176; return size;
-        case 17: size = 55022540; decompressMemory = 4347176; return size;
-        case 18: size = 55171528; decompressMemory = 4347176; return size;
-        case 19: size = 55815954; decompressMemory = 4347176; return size;
-        case 20: size = 55815954; decompressMemory = 4347176; return size;
-        case 21: size = 55815954; decompressMemory = 4347176; return size;
-        case 22: size = 55815954; decompressMemory = 4347176; return size;
+        case 1: size = 824228; decompressMemory = 415024; return size;
+        case 2: size = 1282980; decompressMemory = 415024; return size;
+        case 3: size = 922532; decompressMemory = 415024; return size;
+        case 4: size = 1414052; decompressMemory = 415024; return size;
+        case 5: size = 1545124; decompressMemory = 415024; return size;
+        case 6: size = 1807268; decompressMemory = 415024; return size;
+        case 7: size = 1807268; decompressMemory = 415024; return size;
+        case 8: size = 1807268; decompressMemory = 415024; return size;
+        case 9: size = 1807268; decompressMemory = 415024; return size;
+        case 10: size = 1807268; decompressMemory = 415024; return size;
+        case 11: size = 2331556; decompressMemory = 415024; return size;
+        case 12: size = 2331556; decompressMemory = 415024; return size;
+        case 13: size = 3380132; decompressMemory = 415024; return size;
+        case 14: size = 3004832; decompressMemory = 415024; return size;
+        case 15: size = 3004832; decompressMemory = 415024; return size;
+        case 16: size = 4697834; decompressMemory = 415024; return size;
+        case 17: size = 4697834; decompressMemory = 415024; return size;
+        case 18: size = 4697834; decompressMemory = 415024; return size;
+        case 19: size = 4697834; decompressMemory = 415024; return size;
+        case 20: size = 4697834; decompressMemory = 415024; return size;
+        case 21: size = 4697834; decompressMemory = 415024; return size;
+        case 22: size = 4697834; decompressMemory = 415024; return size;
       }
       decompressMemory = 0;
       return size;

CPP/7zip/zstd.mak

@@ -8,7 +8,6 @@ ZSTD_OBJS = \
   $O\entropy_common.obj \
   $O\fse_decompress.obj \
   $O\huf_decompress.obj \
-  $O\zbuff_decompress.obj \
   $O\zstd_common.obj \
   $O\zstd_decompress.obj \
   $O\xxhash.obj \

CPP/build-x32.cmd

@@ -11,22 +11,10 @@ cd %ROOT%\Bundles\Format7z
 nmake %OPTS%
 copy O\7za.dll %OUTDIR%\7za.dll
 
-cd %ROOT%\Bundles\Format7zExtract
-nmake %OPTS%
-copy O\7zxa.dll %OUTDIR%\7zxa.dll
-
-cd %ROOT%\Bundles\Format7zExtractR
-nmake %OPTS%
-copy O\7zxr.dll %OUTDIR%\7zxr.dll
-
 cd %ROOT%\Bundles\Format7zF
 nmake %OPTS%
 copy O\7z.dll %OUTDIR%\7z.dll
 
-cd %ROOT%\Bundles\Format7zR
-nmake %OPTS%
-copy O\7zra.dll %OUTDIR%\7zra.dll
-
 cd %ROOT%\UI\FileManager
 nmake %OPTS%
 copy O\7zFM.exe %OUTDIR%\7zFM.exe
@@ -43,6 +31,10 @@ cd %ROOT%\Bundles\Codec_zstd
 nmake %OPTS%
 copy O\zstd.dll %OUTDIR%\zstd-x32.dll
 
+cd %ROOT%\Bundles\Codec_zstdF
+nmake %OPTS%
+copy O\zstd.dll %OUTDIR%\zstd-x32big.dll
+
 cd %ROOT%\..\..\C\Util\7zipInstall
 nmake %OPTS%
 copy O\7zipInstall.exe %OUTDIR%\Install-x32.exe

CPP/build-x64.cmd

@@ -11,22 +11,10 @@ cd %ROOT%\Bundles\Format7z
 nmake %OPTS%
 copy AMD64\7za.dll %OUTDIR%\7za.dll
 
-cd %ROOT%\Bundles\Format7zExtract
-nmake %OPTS%
-copy AMD64\7zxa.dll %OUTDIR%\7zxa.dll
-
-cd %ROOT%\Bundles\Format7zExtractR
-nmake %OPTS%
-copy AMD64\7zxr.dll %OUTDIR%\7zxr.dll
-
 cd %ROOT%\Bundles\Format7zF
 nmake %OPTS%
 copy AMD64\7z.dll %OUTDIR%\7z.dll
 
-cd %ROOT%\Bundles\Format7zR
-nmake %OPTS%
-copy AMD64\7zra.dll %OUTDIR%\7zra.dll
-
 cd %ROOT%\UI\FileManager
 nmake %OPTS%
 copy AMD64\7zFM.exe %OUTDIR%\7zFM.exe
@@ -43,6 +31,10 @@ cd %ROOT%\Bundles\Codec_zstd
 nmake %OPTS%
 copy AMD64\zstd.dll %OUTDIR%\zstd-x64.dll
 
+cd %ROOT%\Bundles\Codec_zstdf
+nmake %OPTS%
+copy AMD64\zstd.dll %OUTDIR%\zstd-x64big.dll
+
 cd %ROOT%\..\..\C\Util\7zipInstall
 nmake %OPTS%
 copy AMD64\7zipInstall.exe %OUTDIR%\Install-x64.exe