21.02

C/XzDec.c

@@ -1,5 +1,5 @@
 /* XzDec.c -- Xz Decode
-2019-02-02 : Igor Pavlov : Public domain */
+2021-04-01 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -240,6 +240,7 @@ static SRes BraState_Code2(void *pp,
 }
 
 
+SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc);
 SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc)
 {
   CBraState *decoder;
@@ -1038,7 +1039,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
             (p->outBuf ? NULL : dest), &destLen2, destFinish,
             src, &srcLen2, srcFinished2,
             finishMode2);
-        
+
         *status = p->decoder.status;
         XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2);
         if (!p->outBuf)
@@ -1275,9 +1276,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
         }
         else
         {
+          const Byte *ptr = p->buf;
           p->state = XZ_STATE_STREAM_FOOTER;
           p->pos = 0;
-          if (CRC_GET_DIGEST(p->crc) != GetUi32(p->buf))
+          if (CRC_GET_DIGEST(p->crc) != GetUi32(ptr))
             return SZ_ERROR_CRC;
         }
         break;
@@ -1456,7 +1458,6 @@ typedef struct
   ISeqInStream *inStream;
   ISeqOutStream *outStream;
   ICompressProgress *progress;
-  // CXzStatInfo *stat;
 
   BoolInt finishMode;
   BoolInt outSize_Defined;
@@ -1492,8 +1493,9 @@ typedef struct
   UInt64 numBlocks;
 
   // UInt64 numBadBlocks;
-  SRes mainErrorCode;
-
+  SRes mainErrorCode;  // it's set to error code, if the size Code() output doesn't patch the size from Parsing stage
+                       // it can be = SZ_ERROR_INPUT_EOF
+                       // it can be = SZ_ERROR_DATA, in some another cases
   BoolInt isBlockHeaderState_Parse;
   BoolInt isBlockHeaderState_Write;
   UInt64 outProcessed_Parse;
@@ -1877,7 +1879,7 @@ static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex)
     {
       // if (res == SZ_ERROR_MEM) return res;
       if (me->props.ignoreErrors && res != SZ_ERROR_MEM)
-        return S_OK;
+        return SZ_OK;
       return res;
     }
   }
@@ -1898,15 +1900,18 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex,
   *outCodePos = coder->outCodeSize;
   *stop = True;
 
+  if (srcSize > coder->inPreSize - coder->inCodeSize)
+    return SZ_ERROR_FAIL;
+  
   if (coder->inCodeSize < coder->inPreHeaderSize)
   {
-    UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize;
-    size_t step = srcSize;
-    if (step > rem)
-      step = (size_t)rem;
+    size_t step = coder->inPreHeaderSize - coder->inCodeSize;
+    if (step > srcSize)
+      step = srcSize;
     src += step;
     srcSize -= step;
     coder->inCodeSize += step;
+    *inCodePos = coder->inCodeSize;
     if (coder->inCodeSize < coder->inPreHeaderSize)
     {
       *stop = False;
@@ -1956,7 +1961,7 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex,
   {
     *inCodePos = coder->inPreSize;
     *outCodePos = coder->outPreSize;
-    return S_OK;
+    return SZ_OK;
   }
   return coder->codeRes;
 }
@@ -1966,7 +1971,7 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex,
 
 static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
     BoolInt needWriteToStream,
-    const Byte *src, size_t srcSize,
+    const Byte *src, size_t srcSize, BoolInt isCross,
     // int srcFinished,
     BoolInt *needContinue,
     BoolInt *canRecode)
@@ -1985,7 +1990,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
   if (!coder->dec.headerParsedOk || !coder->outBuf)
   {
     if (me->finishedDecoderIndex < 0)
-      me->finishedDecoderIndex = coderIndex;
+      me->finishedDecoderIndex = (int)coderIndex;
     return SZ_OK;
   }
 
@@ -2077,7 +2082,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
     if (coder->codeRes != SZ_OK)
       if (!me->props.ignoreErrors)
       {
-        me->finishedDecoderIndex = coderIndex;
+        me->finishedDecoderIndex = (int)coderIndex;
         return res;
       }
 
@@ -2086,7 +2091,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
     if (coder->inPreSize != coder->inCodeSize
         || coder->blockPackTotal != coder->inCodeSize)
     {
-      me->finishedDecoderIndex = coderIndex;
+      me->finishedDecoderIndex = (int)coderIndex;
       return SZ_OK;
     }
 
@@ -2125,22 +2130,41 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
         return SZ_OK;
       }
       
+      /*
+      We have processed all xz-blocks of stream,
+      And xz unpacker is at XZ_STATE_BLOCK_HEADER state, where
+      (src) is a pointer to xz-Index structure.
+      We finish reading of current xz-Stream, including Zero padding after xz-Stream.
+      We exit, if we reach extra byte (first byte of new-Stream or another data).
+      But we don't update input stream pointer for that new extra byte.
+      If extra byte is not correct first byte of xz-signature,
+      we have SZ_ERROR_NO_ARCHIVE error here.
+      */
+
       res = XzUnpacker_Code(dec,
           NULL, &outSizeCur,
           src, &srcProcessed,
           me->mtc.readWasFinished, // srcFinished
           CODER_FINISH_END, // CODER_FINISH_ANY,
           &status);
+
+      // res = SZ_ERROR_ARCHIVE; // for failure test
       
       me->status = status;
       me->codeRes = res;
 
+      if (isCross)
+        me->mtc.crossStart += srcProcessed;
+
       me->mtc.inProcessed += srcProcessed;
       me->mtc.mtProgress.totalInSize = me->mtc.inProcessed;
 
+      srcSize -= srcProcessed;
+      src += srcProcessed;
+
       if (res != SZ_OK)
       {
-        return S_OK;
+        return SZ_OK;
         // return res;
       }
       
@@ -2149,20 +2173,26 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
         *needContinue = True;
         me->isBlockHeaderState_Parse = False;
         me->isBlockHeaderState_Write = False;
+
+        if (!isCross)
         {
           Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc);
           if (!crossBuf)
             return SZ_ERROR_MEM;
-          memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed);
+          if (srcSize != 0)
+            memcpy(crossBuf, src, srcSize);
+          me->mtc.crossStart = 0;
+          me->mtc.crossEnd = srcSize;
         }
-        me->mtc.crossStart = 0;
-        me->mtc.crossEnd = srcSize - srcProcessed;
+
+        PRF_STR_INT("XZ_STATE_STREAM_HEADER crossEnd = ", (unsigned)me->mtc.crossEnd);
+
         return SZ_OK;
       }
       
-      if (status != CODER_STATUS_NEEDS_MORE_INPUT)
+      if (status != CODER_STATUS_NEEDS_MORE_INPUT || srcSize != 0)
       {
-        return E_FAIL;
+        return SZ_ERROR_FAIL;
       }
       
       if (me->mtc.readWasFinished)
@@ -2174,7 +2204,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
     {
       size_t inPos;
       size_t inLim;
-      const Byte *inData;
+      // const Byte *inData;
       UInt64 inProgressPrev = me->mtc.inProcessed;
       
       // XzDecMt_Prepare_InBuf_ST(p);
@@ -2184,9 +2214,8 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
       
       inPos = 0;
       inLim = 0;
-      // outProcessed = 0;
       
-      inData = crossBuf;
+      // inData = crossBuf;
       
       for (;;)
       {
@@ -2201,7 +2230,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
           {
             inPos = 0;
             inLim = me->mtc.inBufSize;
-            me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim);
+            me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)crossBuf, &inLim);
             me->mtc.readProcessed += inLim;
             if (inLim == 0 || me->mtc.readRes != SZ_OK)
               me->mtc.readWasFinished = True;
@@ -2213,7 +2242,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
 
         res = XzUnpacker_Code(dec,
             NULL, &outProcessed,
-            inData + inPos, &inProcessed,
+            crossBuf + inPos, &inProcessed,
             (inProcessed == 0), // srcFinished
             CODER_FINISH_END, &status);
         
@@ -2225,7 +2254,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
 
         if (res != SZ_OK)
         {
-          return S_OK;
+          return SZ_OK;
           // return res;
         }
 
@@ -2240,7 +2269,7 @@ static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
         }
         
         if (status != CODER_STATUS_NEEDS_MORE_INPUT)
-          return E_FAIL;
+          return SZ_ERROR_FAIL;
         
         if (me->mtc.progress)
         {
@@ -2276,13 +2305,6 @@ void XzStatInfo_Clear(CXzStatInfo *p)
   p->NumStreams_Defined = False;
   p->NumBlocks_Defined = False;
   
-  // p->IsArc = False;
-  // p->UnexpectedEnd = False;
-  // p->Unsupported = False;
-  // p->HeadersError = False;
-  // p->DataError = False;
-  // p->CrcError = False;
-
   p->DataAfterEnd = False;
   p->DecodingTruncated = False;
   
@@ -2296,6 +2318,16 @@ void XzStatInfo_Clear(CXzStatInfo *p)
 
 
 
+/*
+  XzDecMt_Decode_ST() can return SZ_OK or the following errors
+     - SZ_ERROR_MEM for memory allocation error
+     - error from XzUnpacker_Code() function
+     - SZ_ERROR_WRITE for ISeqOutStream::Write(). stat->CombinedRes_Type = SZ_ERROR_WRITE in that case
+     - ICompressProgress::Progress() error,  stat->CombinedRes_Type = SZ_ERROR_PROGRESS.
+  But XzDecMt_Decode_ST() doesn't return ISeqInStream::Read() errors.
+  ISeqInStream::Read() result is set to p->readRes.
+  also it can set stat->CombinedRes_Type to SZ_ERROR_WRITE or SZ_ERROR_PROGRESS.
+*/
 
 static SRes XzDecMt_Decode_ST(CXzDecMt *p
     #ifndef _7ZIP_ST
@@ -2384,7 +2416,7 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p
         inPos = 0;
         inLim = p->inBufSize;
         inData = p->inBuf;
-        p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim);
+        p->readRes = ISeqInStream_Read(p->inStream, (void *)p->inBuf, &inLim);
         p->readProcessed += inLim;
         if (inLim == 0 || p->readRes != SZ_OK)
           p->readWasFinished = True;
@@ -2426,8 +2458,8 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p
     if (finished || outProcessed >= outSize)
       if (outPos != 0)
       {
-        size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos);
-        p->outProcessed += written;
+        const size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos);
+        // p->outProcessed += written; // 21.01: BUG fixed
         if (written != outPos)
         {
           stat->CombinedRes_Type = SZ_ERROR_WRITE;
@@ -2438,9 +2470,8 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p
 
     if (p->progress && res == SZ_OK)
     {
-      UInt64 inDelta = p->inProcessed - inPrev;
-      UInt64 outDelta = p->outProcessed - outPrev;
-      if (inDelta >= (1 << 22) || outDelta >= (1 << 22))
+      if (p->inProcessed - inPrev >= (1 << 22) ||
+          p->outProcessed - outPrev >= (1 << 22))
       {
         res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed);
         if (res != SZ_OK)
@@ -2455,14 +2486,31 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p
     }
 
     if (finished)
-      return res;
+    {
+      // p->codeRes is preliminary error from XzUnpacker_Code.
+      // and it can be corrected later as final result
+      // so we return SZ_OK here instead of (res);
+      return SZ_OK;
+      // return res;
+    }
   }
 }
 
-static SRes XzStatInfo_SetStat(const CXzUnpacker *dec,
+
+
+/*
+XzStatInfo_SetStat() transforms
+    CXzUnpacker return code and status to combined CXzStatInfo results.
+    it can convert SZ_OK to SZ_ERROR_INPUT_EOF
+    it can convert SZ_ERROR_NO_ARCHIVE to SZ_OK and (DataAfterEnd = 1)
+*/
+
+static void XzStatInfo_SetStat(const CXzUnpacker *dec,
     int finishMode,
-    UInt64 readProcessed, UInt64 inProcessed,
-    SRes res, ECoderStatus status,
+    // UInt64 readProcessed,
+    UInt64 inProcessed,
+    SRes res,                     // it's result from CXzUnpacker unpacker
+    ECoderStatus status,
     BoolInt decodingTruncated,
     CXzStatInfo *stat)
 {
@@ -2484,12 +2532,20 @@ static SRes XzStatInfo_SetStat(const CXzUnpacker *dec,
     if (status == CODER_STATUS_NEEDS_MORE_INPUT)
     {
       // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams
+      // any extra data is part of correct data
       extraSize = 0;
+      // if xz stream was not finished, then we need more data
       if (!XzUnpacker_IsStreamWasFinished(dec))
         res = SZ_ERROR_INPUT_EOF;
     }
-    else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED)
-      res = SZ_ERROR_DATA;
+    else
+    {
+      // CODER_STATUS_FINISHED_WITH_MARK is not possible for multi stream xz decoding
+      // so he we have (status == CODER_STATUS_NOT_FINISHED)
+      // if (status != CODER_STATUS_FINISHED_WITH_MARK)
+      if (!decodingTruncated || finishMode)
+        res = SZ_ERROR_DATA;
+    }
   }
   else if (res == SZ_ERROR_NO_ARCHIVE)
   {
@@ -2497,24 +2553,29 @@ static SRes XzStatInfo_SetStat(const CXzUnpacker *dec,
     SZ_ERROR_NO_ARCHIVE is possible for 2 states:
       XZ_STATE_STREAM_HEADER  - if bad signature or bad CRC
       XZ_STATE_STREAM_PADDING - if non-zero padding data
-    extraSize / inProcessed don't include "bad" byte
+    extraSize and inProcessed don't include "bad" byte
     */
-    if (inProcessed != extraSize) // if good streams before error
-      if (extraSize != 0 || readProcessed != inProcessed)
+    // if (inProcessed == extraSize), there was no any good xz stream header, and we keep error
+    if (inProcessed != extraSize) // if there were good xz streams before error
+    {
+      // if (extraSize != 0 || readProcessed != inProcessed)
       {
+        // he we suppose that all xz streams were finsihed OK, and we have
+        // some extra data after all streams
         stat->DataAfterEnd = True;
-        // there is some good xz stream before. So we set SZ_OK
         res = SZ_OK;
       }
+    }
   }
   
-  stat->DecodeRes = res;
+  if (stat->DecodeRes == SZ_OK)
+    stat->DecodeRes = res;
 
   stat->InSize -= extraSize;
-  return res;
 }
 
 
+
 SRes XzDecMt_Decode(CXzDecMtHandle pp,
     const CXzDecMtProps *props,
     const UInt64 *outDataSize, int finishMode,
@@ -2557,8 +2618,9 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
   p->inProcessed = 0;
   p->readProcessed = 0;
   p->readWasFinished = False;
+  p->readRes = SZ_OK;
 
-  p->codeRes = 0;
+  p->codeRes = SZ_OK;
   p->status = CODER_STATUS_NOT_SPECIFIED;
 
   XzUnpacker_Init(&p->dec);
@@ -2589,8 +2651,9 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
 
   if (p->props.numThreads > 1)
   {
-    IMtDecCallback vt;
-
+    IMtDecCallback2 vt;
+    BoolInt needContinue;
+    SRes res;
     // we just free ST buffers here
     // but we still keep state variables, that was set in XzUnpacker_Init()
     XzDecMt_FreeSt(p);
@@ -2628,45 +2691,45 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
     vt.Code = XzDecMt_Callback_Code;
     vt.Write = XzDecMt_Callback_Write;
 
+
+    res = MtDec_Code(&p->mtc);
+
+
+    stat->InSize = p->mtc.inProcessed;
+    
+    p->inProcessed = p->mtc.inProcessed;
+    p->readRes = p->mtc.readRes;
+    p->readWasFinished = p->mtc.readWasFinished;
+    p->readProcessed = p->mtc.readProcessed;
+    
+    tMode = True;
+    needContinue = False;
+    
+    if (res == SZ_OK)
     {
-      BoolInt needContinue;
-      
-      SRes res = MtDec_Code(&p->mtc);
-
-      stat->InSize = p->mtc.inProcessed;
-
-      p->inProcessed = p->mtc.inProcessed;
-      p->readRes = p->mtc.readRes;
-      p->readWasFinished = p->mtc.readWasFinished;
-      p->readProcessed = p->mtc.readProcessed;
-
-      tMode = True;
-      needContinue = False;
-
-      if (res == SZ_OK)
+      if (p->mtc.mtProgress.res != SZ_OK)
       {
-        if (p->mtc.mtProgress.res != SZ_OK)
-        {
-          res = p->mtc.mtProgress.res;
-          stat->ProgressRes = res;
-          stat->CombinedRes_Type = SZ_ERROR_PROGRESS;
-        }
-        else
-          needContinue = p->mtc.needContinue;
+        res = p->mtc.mtProgress.res;
+        stat->ProgressRes = res;
+        stat->CombinedRes_Type = SZ_ERROR_PROGRESS;
       }
-
-      if (!needContinue)
+      else
+        needContinue = p->mtc.needContinue;
+    }
+    
+    if (!needContinue)
+    {
       {
         SRes codeRes;
         BoolInt truncated = False;
         ECoderStatus status;
-        CXzUnpacker *dec;
+        const CXzUnpacker *dec;
 
         stat->OutSize = p->outProcessed;
        
         if (p->finishedDecoderIndex >= 0)
         {
-          CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex];
+          const CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex];
           codeRes = coder->codeRes;
           dec = &coder->dec;
           status = coder->status;
@@ -2679,41 +2742,46 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
           truncated = p->parsing_Truncated;
         }
         else
-          return E_FAIL;
+          return SZ_ERROR_FAIL;
+
+        if (p->mainErrorCode != SZ_OK)
+          stat->DecodeRes = p->mainErrorCode;
 
         XzStatInfo_SetStat(dec, p->finishMode,
-            p->mtc.readProcessed, p->mtc.inProcessed,
+            // p->mtc.readProcessed,
+            p->mtc.inProcessed,
             codeRes, status,
             truncated,
             stat);
-
-        if (res == SZ_OK)
-        {
-          if (p->writeRes != SZ_OK)
-          {
-            res = p->writeRes;
-            stat->CombinedRes_Type = SZ_ERROR_WRITE;
-          }
-          else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed)
-          {
-            res = p->mtc.readRes;
-            stat->ReadRes = res;
-            stat->CombinedRes_Type = SZ_ERROR_READ;
-          }
-          else if (p->mainErrorCode != SZ_OK)
-          {
-            res = p->mainErrorCode;
-          }
-        }
-
-        stat->CombinedRes = res;
-        if (stat->CombinedRes_Type == SZ_OK)
-          stat->CombinedRes_Type = res;
-        return res;
       }
 
-      PRF_STR("----- decoding ST -----");
+      if (res == SZ_OK)
+      {
+        stat->ReadRes = p->mtc.readRes;
+
+        if (p->writeRes != SZ_OK)
+        {
+          res = p->writeRes;
+          stat->CombinedRes_Type = SZ_ERROR_WRITE;
+        }
+        else if (p->mtc.readRes != SZ_OK
+            // && p->mtc.inProcessed == p->mtc.readProcessed
+            && stat->DecodeRes == SZ_ERROR_INPUT_EOF)
+        {
+          res = p->mtc.readRes;
+          stat->CombinedRes_Type = SZ_ERROR_READ;
+        }
+        else if (stat->DecodeRes != SZ_OK)
+          res = stat->DecodeRes;
+      }
+      
+      stat->CombinedRes = res;
+      if (stat->CombinedRes_Type == SZ_OK)
+        stat->CombinedRes_Type = res;
+      return res;
     }
+
+    PRF_STR("----- decoding ST -----");
   }
 
   #endif
@@ -2729,33 +2797,35 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
         , stat
         );
 
+    #ifndef _7ZIP_ST
+    // we must set error code from MT decoding at first
+    if (p->mainErrorCode != SZ_OK)
+      stat->DecodeRes = p->mainErrorCode;
+    #endif
+
     XzStatInfo_SetStat(&p->dec,
         p->finishMode,
-        p->readProcessed, p->inProcessed,
+        // p->readProcessed,
+        p->inProcessed,
         p->codeRes, p->status,
         False, // truncated
         stat);
 
+    stat->ReadRes = p->readRes;
+
     if (res == SZ_OK)
     {
-      /*
-      if (p->writeRes != SZ_OK)
-      {
-        res = p->writeRes;
-        stat->CombinedRes_Type = SZ_ERROR_WRITE;
-      }
-      else
-      */
-      if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed)
+      if (p->readRes != SZ_OK
+          // && p->inProcessed == p->readProcessed
+          && stat->DecodeRes == SZ_ERROR_INPUT_EOF)
       {
+        // we set read error as combined error, only if that error was the reason
+        // of decoding problem
         res = p->readRes;
-        stat->ReadRes = res;
         stat->CombinedRes_Type = SZ_ERROR_READ;
       }
-      #ifndef _7ZIP_ST
-      else if (p->mainErrorCode != SZ_OK)
-        res = p->mainErrorCode;
-      #endif
+      else if (stat->DecodeRes != SZ_OK)
+        res = stat->DecodeRes;
     }
 
     stat->CombinedRes = res;