Merge pull request #58 from conor42/master

Add Fast LZMA2 codec

CPP/7zip/Compress/Lzma2Encoder.cpp

@@ -4,6 +4,8 @@
 
 #include "../../../C/Alloc.h"
 
+#include "../../../C/fast-lzma2/fl2_errors.h"
+
 #include "../Common/CWrappers.h"
 #include "../Common/StreamUtils.h"
 
@@ -119,4 +121,167 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream
   return SResToHRESULT(res);
 }
   
+CFastEncoder::CFastEncoder()
+{
+  _encoder = NULL;
+  reduceSize = 0;
+}
+
+CFastEncoder::~CFastEncoder()
+{
+  if (_encoder)
+    FL2_freeCCtx(_encoder);
+}
+
+
+#define CHECK_F(f) if (FL2_isError(f)) return E_INVALIDARG;  /* check and convert error code */
+
+STDMETHODIMP CFastEncoder::SetCoderProperties(const PROPID *propIDs,
+  const PROPVARIANT *coderProps, UInt32 numProps)
+{
+  CLzma2EncProps lzma2Props;
+  Lzma2EncProps_Init(&lzma2Props);
+
+  for (UInt32 i = 0; i < numProps; i++)
+  {
+    RINOK(SetLzma2Prop(propIDs[i], coderProps[i], lzma2Props));
+  }
+  if (_encoder == NULL) {
+    _encoder = FL2_createCCtxMt(lzma2Props.numTotalThreads);
+    if (_encoder == NULL)
+      return E_OUTOFMEMORY;
+  }
+  if (lzma2Props.lzmaProps.algo > 2) {
+    if (lzma2Props.lzmaProps.algo > 3)
+      return E_INVALIDARG;
+    lzma2Props.lzmaProps.algo = 2;
+    FL2_CCtx_setParameter(_encoder, FL2_p_highCompression, 1);
+    FL2_CCtx_setParameter(_encoder, FL2_p_compressionLevel, lzma2Props.lzmaProps.level);
+  }
+  else {
+    FL2_CCtx_setParameter(_encoder, FL2_p_7zLevel, lzma2Props.lzmaProps.level);
+  }
+  dictSize = lzma2Props.lzmaProps.dictSize;
+  if (!dictSize) {
+    dictSize = (UInt32)1 << FL2_CCtx_setParameter(_encoder, FL2_p_dictionaryLog, 0);
+  }
+  reduceSize = lzma2Props.lzmaProps.reduceSize;
+  reduceSize += (reduceSize < (UInt64)-1); /* prevent extra buffer shift after read */
+  dictSize = (UInt32)min(dictSize, reduceSize);
+  unsigned dictLog = FL2_DICTLOG_MIN;
+  while (((UInt32)1 << dictLog) < dictSize)
+    ++dictLog;
+  CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_dictionaryLog, dictLog));
+  if (lzma2Props.lzmaProps.algo >= 0) {
+    CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_strategy, (unsigned)lzma2Props.lzmaProps.algo));
+  }
+  if (lzma2Props.lzmaProps.fb > 0)
+    CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_fastLength, lzma2Props.lzmaProps.fb));
+  if (lzma2Props.lzmaProps.mc) {
+    unsigned ml = 0;
+    while (((UInt32)1 << ml) < lzma2Props.lzmaProps.mc)
+      ++ml;
+    CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_searchLog, ml));
+  }
+  if (lzma2Props.lzmaProps.lc >= 0)
+    CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_literalCtxBits, lzma2Props.lzmaProps.lc));
+  if (lzma2Props.lzmaProps.lp >= 0)
+    CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_literalPosBits, lzma2Props.lzmaProps.lp));
+  if (lzma2Props.lzmaProps.pb >= 0)
+    CHECK_F(FL2_CCtx_setParameter(_encoder, FL2_p_posBits, lzma2Props.lzmaProps.pb));
+  FL2_CCtx_setParameter(_encoder, FL2_p_omitProperties, 1);
+  FL2_CCtx_setParameter(_encoder, FL2_p_doXXHash, 0);
+  return S_OK;
+}
+
+
+#define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
+
+STDMETHODIMP CFastEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
+{
+  Byte prop;
+  unsigned i;
+  for (i = 0; i < 40; i++)
+    if (dictSize <= LZMA2_DIC_SIZE_FROM_PROP(i))
+      break;
+  prop = (Byte)i;
+  return WriteStream(outStream, &prop, 1);
+}
+
+
+typedef struct
+{
+  ISequentialOutStream* outStream;
+  ICompressProgressInfo* progress;
+  UInt64 in_processed;
+  UInt64 out_processed;
+  HRESULT res;
+} EncodingObjects;
+
+static int FL2LIB_CALL Progress(size_t done, void* opaque)
+{
+  EncodingObjects* p = (EncodingObjects*)opaque;
+  if (p && p->progress) {
+    UInt64 in_processed = p->in_processed + done;
+    p->res = p->progress->SetRatioInfo(&in_processed, &p->out_processed);
+    return p->res != S_OK;
+  }
+  return 0;
+}
+
+static int FL2LIB_CALL Write(const void* src, size_t srcSize, void* opaque)
+{
+  EncodingObjects* p = (EncodingObjects*)opaque;
+  p->res = WriteStream(p->outStream, src, srcSize);
+  return p->res != S_OK;
+}
+
+STDMETHODIMP CFastEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
+  const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
+{
+  HRESULT err = S_OK;
+  inBuffer.AllocAtLeast(dictSize);
+  EncodingObjects objs = { outStream, progress, 0, 0, S_OK };
+  FL2_blockBuffer block = { inBuffer, 0, 0, dictSize };
+  do
+  {
+    FL2_shiftBlock(_encoder, &block);
+    size_t inSize = dictSize - block.start;
+    err = ReadStream(inStream, inBuffer + block.start, &inSize);
+    if (err != S_OK)
+      break;
+    block.end += inSize;
+    if (inSize) {
+      size_t cSize = FL2_compressCCtxBlock_toFn(_encoder, Write, &objs, &block, Progress);
+      if (FL2_isError(cSize)) {
+        if (FL2_getErrorCode(cSize) == FL2_error_memory_allocation)
+          return E_OUTOFMEMORY;
+        return objs.res != S_OK ? objs.res : S_FALSE;
+      }
+      if (objs.res != S_OK)
+        return objs.res;
+      objs.out_processed += cSize;
+      objs.in_processed += inSize;
+      if (progress) {
+        err = progress->SetRatioInfo(&objs.in_processed, &objs.out_processed);
+        if (err != S_OK)
+          break;
+      }
+      if (block.end < dictSize)
+        break;
+    }
+    else break;
+
+  } while (err == S_OK);
+
+  if (err == S_OK) {
+    size_t cSize = FL2_endFrame_toFn(_encoder, Write, &objs);
+    if (FL2_isError(cSize))
+      return S_FALSE;
+    objs.out_processed += cSize;
+    err = objs.res;
+  }
+  return err;
+}
+
 }}

CPP/7zip/Compress/Lzma2Encoder.h

@@ -4,8 +4,10 @@
 #define __LZMA2_ENCODER_H
 
 #include "../../../C/Lzma2Enc.h"
+#include "../../../C/fast-lzma2/fast-lzma2.h"
 
 #include "../../Common/MyCom.h"
+#include "../../Common/MyBuffer.h"
 
 #include "../ICoder.h"
 
@@ -37,6 +39,32 @@ public:
   virtual ~CEncoder();
 };
 
+class CFastEncoder :
+  public ICompressCoder,
+  public ICompressSetCoderProperties,
+  public ICompressWriteCoderProperties,
+  public CMyUnknownImp
+{
+  FL2_CCtx* _encoder;
+  CByteBuffer inBuffer;
+  UInt64 reduceSize;
+  UInt32 dictSize;
+
+public:
+  MY_UNKNOWN_IMP3(
+    ICompressCoder,
+    ICompressSetCoderProperties,
+    ICompressWriteCoderProperties)
+
+    STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
+      const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
+  STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
+  STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
+
+  CFastEncoder();
+  virtual ~CFastEncoder();
+};
+
 }}
 
 #endif

CPP/7zip/Compress/Lzma2Register.cpp

@@ -14,9 +14,19 @@ namespace NCompress {
 namespace NLzma2 {
 
 REGISTER_CODEC_E(LZMA2,
-    CDecoder(),
-    CEncoder(),
-    0x21,
-    "LZMA2")
+  CDecoder(),
+  CEncoder(),
+  0x21,
+  "LZMA2")
+}
 
-}}
+namespace NFLzma2 {
+
+REGISTER_CODEC_E(FLZMA2,
+  NCompress::NLzma2::CDecoder(),
+  NCompress::NLzma2::CFastEncoder(),
+  0x21,
+  "FLZMA2")
+}
+
+}