18.03

Asm/x86/7zAsm.asm

@@ -1,5 +1,5 @@
 ; 7zAsm.asm -- ASM macros
-; 2012-12-30 : Igor Pavlov : Public domain
+; 2018-02-03 : Igor Pavlov : Public domain
 
 MY_ASM_START macro
   ifdef x64
@@ -52,6 +52,15 @@ endif
   x6 equ ESI
   x7 equ EDI
 
+  x0_W equ AX
+  x1_W equ CX
+  x2_W equ DX
+  x3_W equ BX
+
+  x5_W equ BP
+  x6_W equ SI
+  x7_W equ DI
+
   x0_L equ AL
   x1_L equ CL
   x2_L equ DL
@@ -63,6 +72,10 @@ endif
   x3_H equ BH
 
 ifdef x64
+  x5_L equ BPL
+  x6_L equ SIL
+  x7_L equ DIL
+
   r0 equ RAX
   r1 equ RCX
   r2 equ RDX
@@ -103,3 +116,32 @@ MY_POP_4_REGS macro
     pop     r5
     pop     r3
 endm
+
+
+ifdef x64
+
+; for WIN64-x64 ABI:
+
+REG_PARAM_0 equ r1
+REG_PARAM_1 equ r2
+REG_PARAM_2 equ r8
+REG_PARAM_3 equ r9
+
+MY_PUSH_PRESERVED_REGS macro
+    MY_PUSH_4_REGS
+    push    r12
+    push    r13
+    push    r14
+    push    r15
+endm
+
+
+MY_POP_PRESERVED_REGS macro
+    pop     r15
+    pop     r14
+    pop     r13
+    pop     r12
+    MY_POP_4_REGS
+endm
+
+endif

C/7zCrc.c

@@ -1,5 +1,5 @@
 /* 7zCrc.c -- CRC32 init
-2017-04-03 : Igor Pavlov : Public domain */
+2017-06-06 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -86,8 +86,8 @@ void MY_FAST_CALL CrcGenerateTable()
   
       #ifdef MY_CPU_X86_OR_AMD64
       if (!CPU_Is_InOrder())
-        g_CrcUpdate = CrcUpdateT8;
       #endif
+        g_CrcUpdate = CrcUpdateT8;
     #endif
 
   #else
@@ -101,7 +101,7 @@ void MY_FAST_CALL CrcGenerateTable()
       g_CrcUpdate = CrcUpdateT4;
       #if CRC_NUM_TABLES >= 8
       g_CrcUpdateT8 = CrcUpdateT8;
-      // g_CrcUpdate = CrcUpdateT8;
+      g_CrcUpdate = CrcUpdateT8;
       #endif
     }
     else if (p[0] != 1 || p[1] != 2)
@@ -118,7 +118,7 @@ void MY_FAST_CALL CrcGenerateTable()
       g_CrcUpdate = CrcUpdateT1_BeT4;
       #if CRC_NUM_TABLES >= 8
       g_CrcUpdateT8 = CrcUpdateT1_BeT8;
-      // g_CrcUpdate = CrcUpdateT1_BeT8;
+      g_CrcUpdate = CrcUpdateT1_BeT8;
       #endif
     }
   }

C/7zTypes.h

@@ -1,5 +1,5 @@
 /* 7zTypes.h -- Basic types
-2017-04-03 : Igor Pavlov : Public domain */
+2017-07-17 : Igor Pavlov : Public domain */
 
 #ifndef __7Z_TYPES_H
 #define __7Z_TYPES_H
@@ -42,13 +42,23 @@ EXTERN_C_BEGIN
 
 typedef int SRes;
 
+
 #ifdef _WIN32
+
 /* typedef DWORD WRes; */
 typedef unsigned WRes;
+#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x)
+
 #else
+
 typedef int WRes;
+#define MY__FACILITY_WIN32 7
+#define MY__FACILITY__WRes MY__FACILITY_WIN32
+#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000)))
+
 #endif
 
+
 #ifndef RINOK
 #define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
 #endif

C/7zVersion.h

@@ -1,7 +1,7 @@
-#define MY_VER_MAJOR 17
-#define MY_VER_MINOR 00
+#define MY_VER_MAJOR 18
+#define MY_VER_MINOR 03
 #define MY_VER_BUILD 0
-#define MY_VERSION_NUMBERS "17.00 beta"
+#define MY_VERSION_NUMBERS "18.03 beta"
 #define MY_VERSION MY_VERSION_NUMBERS
 
 #ifdef MY_CPU_NAME
@@ -10,12 +10,12 @@
   #define MY_VERSION_CPU MY_VERSION
 #endif
 
-#define MY_DATE "2017-04-29"
+#define MY_DATE "2018-03-04"
 #undef MY_COPYRIGHT
 #undef MY_VERSION_COPYRIGHT_DATE
 #define MY_AUTHOR_NAME "Igor Pavlov"
 #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
-#define MY_COPYRIGHT_CR "Copyright (c) 1999-2017 Igor Pavlov"
+#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov"
 
 #ifdef USE_COPYRIGHT_CR
   #define MY_COPYRIGHT MY_COPYRIGHT_CR

C/AesOpt.c

@@ -1,12 +1,12 @@
 /* AesOpt.c -- Intel's AES
-2013-11-12 : Igor Pavlov : Public domain */
+2017-06-08 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
 #include "CpuArch.h"
 
 #ifdef MY_CPU_X86_OR_AMD64
-#if _MSC_VER >= 1500
+#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729)
 #define USE_INTEL_AES
 #endif
 #endif

C/Alloc.c

@@ -1,8 +1,10 @@
 /* Alloc.c -- Memory allocation functions
-2017-04-03 : Igor Pavlov : Public domain */
+2018-03-01 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
+#include <stdio.h>
+
 #ifdef _WIN32
 #include <windows.h>
 #endif
@@ -14,20 +16,127 @@
 
 /* use _SZ_ALLOC_DEBUG to debug alloc/free operations */
 #ifdef _SZ_ALLOC_DEBUG
+
 #include <stdio.h>
 int g_allocCount = 0;
 int g_allocCountMid = 0;
 int g_allocCountBig = 0;
+
+
+#define CONVERT_INT_TO_STR(charType, tempSize) \
+  unsigned char temp[tempSize]; unsigned i = 0; \
+  while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \
+  *s++ = (charType)('0' + (unsigned)val); \
+  while (i != 0) { i--; *s++ = temp[i]; } \
+  *s = 0;
+
+static void ConvertUInt64ToString(UInt64 val, char *s)
+{
+  CONVERT_INT_TO_STR(char, 24);
+}
+
+#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10)))))
+
+static void ConvertUInt64ToHex(UInt64 val, char *s)
+{
+  UInt64 v = val;
+  unsigned i;
+  for (i = 1;; i++)
+  {
+    v >>= 4;
+    if (v == 0)
+      break;
+  }
+  s[i] = 0;
+  do
+  {
+    unsigned t = (unsigned)(val & 0xF);
+    val >>= 4;
+    s[--i] = GET_HEX_CHAR(t);
+  }
+  while (i);
+}
+
+#define DEBUG_OUT_STREAM stderr
+
+static void Print(const char *s)
+{
+  fputs(s, DEBUG_OUT_STREAM);
+}
+
+static void PrintAligned(const char *s, size_t align)
+{
+  size_t len = strlen(s);
+  for(;;)
+  {
+    fputc(' ', DEBUG_OUT_STREAM);
+    if (len >= align)
+      break;
+    ++len;
+  }
+  Print(s);
+}
+
+static void PrintLn()
+{
+  Print("\n");
+}
+
+static void PrintHex(UInt64 v, size_t align)
+{
+  char s[32];
+  ConvertUInt64ToHex(v, s);
+  PrintAligned(s, align);
+}
+
+static void PrintDec(UInt64 v, size_t align)
+{
+  char s[32];
+  ConvertUInt64ToString(v, s);
+  PrintAligned(s, align);
+}
+
+static void PrintAddr(void *p)
+{
+  PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12);
+}
+
+
+#define PRINT_ALLOC(name, cnt, size, ptr) \
+    Print(name " "); \
+    PrintDec(cnt++, 10); \
+    PrintHex(size, 10); \
+    PrintAddr(ptr); \
+    PrintLn();
+ 
+#define PRINT_FREE(name, cnt, ptr) if (ptr) { \
+    Print(name " "); \
+    PrintDec(--cnt, 10); \
+    PrintAddr(ptr); \
+    PrintLn(); }
+ 
+#else
+
+#define PRINT_ALLOC(name, cnt, size, ptr)
+#define PRINT_FREE(name, cnt, ptr)
+#define Print(s)
+#define PrintLn()
+#define PrintHex(v, align)
+#define PrintDec(v, align)
+#define PrintAddr(p)
+
 #endif
 
+
+
 void *MyAlloc(size_t size)
 {
   if (size == 0)
-    return 0;
+    return NULL;
   #ifdef _SZ_ALLOC_DEBUG
   {
     void *p = malloc(size);
-    fprintf(stderr, "\nAlloc %10d bytes, count = %10d,  addr = %8X", size, g_allocCount++, (unsigned)p);
+    PRINT_ALLOC("Alloc    ", g_allocCount, size, p);
     return p;
   }
   #else
@@ -37,10 +146,8 @@ void *MyAlloc(size_t size)
 
 void MyFree(void *address)
 {
-  #ifdef _SZ_ALLOC_DEBUG
-  if (address != 0)
-    fprintf(stderr, "\nFree; count = %10d,  addr = %8X", --g_allocCount, (unsigned)address);
-  #endif
+  PRINT_FREE("Free    ", g_allocCount, address);
+  
   free(address);
 }
 
@@ -49,20 +156,18 @@ void MyFree(void *address)
 void *MidAlloc(size_t size)
 {
   if (size == 0)
-    return 0;
-  #ifdef _SZ_ALLOC_DEBUG
-  fprintf(stderr, "\nAlloc_Mid %10d bytes;  count = %10d", size, g_allocCountMid++);
-  #endif
-  return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
+    return NULL;
+  
+  PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL);
+  
+  return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
 }
 
 void MidFree(void *address)
 {
-  #ifdef _SZ_ALLOC_DEBUG
-  if (address != 0)
-    fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid);
-  #endif
-  if (address == 0)
+  PRINT_FREE("Free-Mid", g_allocCountMid, address);
+
+  if (!address)
     return;
   VirtualFree(address, 0, MEM_RELEASE);
 }
@@ -79,10 +184,10 @@ typedef SIZE_T (WINAPI *GetLargePageMinimumP)();
 void SetLargePageSize()
 {
   #ifdef _7ZIP_LARGE_PAGES
-  SIZE_T size = 0;
+  SIZE_T size;
   GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP)
         GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum");
-  if (largePageMinimum == 0)
+  if (!largePageMinimum)
     return;
   size = largePageMinimum();
   if (size == 0 || (size & (size - 1)) != 0)
@@ -95,31 +200,36 @@ void SetLargePageSize()
 void *BigAlloc(size_t size)
 {
   if (size == 0)
-    return 0;
-  #ifdef _SZ_ALLOC_DEBUG
-  fprintf(stderr, "\nAlloc_Big %10d bytes;  count = %10d", size, g_allocCountBig++);
-  #endif
+    return NULL;
+
+  PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL);
   
   #ifdef _7ZIP_LARGE_PAGES
-  if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18))
   {
-    void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)),
-        MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
-    if (res != 0)
-      return res;
+    SIZE_T ps = g_LargePageSize;
+    if (ps != 0 && ps <= (1 << 30) && size > (ps / 2))
+    {
+      size_t size2;
+      ps--;
+      size2 = (size + ps) & ~ps;
+      if (size2 >= size)
+      {
+        void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
+        if (res)
+          return res;
+      }
+    }
   }
   #endif
-  return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
+
+  return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
 }
 
 void BigFree(void *address)
 {
-  #ifdef _SZ_ALLOC_DEBUG
-  if (address != 0)
-    fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig);
-  #endif
+  PRINT_FREE("Free-Big", g_allocCountBig, address);
   
-  if (address == 0)
+  if (!address)
     return;
   VirtualFree(address, 0, MEM_RELEASE);
 }
@@ -129,8 +239,217 @@ void BigFree(void *address)
 
 static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); }
 static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); }
-ISzAlloc const g_Alloc = { SzAlloc, SzFree };
+const ISzAlloc g_Alloc = { SzAlloc, SzFree };
+
+static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); }
+static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); }
+const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree };
 
 static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); }
 static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); }
-ISzAlloc const g_BigAlloc = { SzBigAlloc, SzBigFree };
+const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
+
+
+/*
+  uintptr_t : <stdint.h> C99 (optional)
+            : unsupported in VS6
+*/
+
+#ifdef _WIN32
+  typedef UINT_PTR UIntPtr;
+#else
+  /*
+  typedef uintptr_t UIntPtr;
+  */
+  typedef ptrdiff_t UIntPtr;
+#endif
+
+
+#define ADJUST_ALLOC_SIZE 0
+/*
+#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1)
+*/
+/*
+  Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if
+     MyAlloc() can return address that is NOT multiple of sizeof(void *).
+*/
+
+
+/*
+#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1))))
+*/
+#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1))))
+
+#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align)
+
+
+#if (_POSIX_C_SOURCE >= 200112L)
+  #define USE_posix_memalign
+#endif
+
+/*
+  This posix_memalign() is for test purposes only.
+  We also need special Free() function instead of free(),
+  if this posix_memalign() is used.
+*/
+
+/*
+static int posix_memalign(void **ptr, size_t align, size_t size)
+{
+  size_t newSize = size + align;
+  void *p;
+  void *pAligned;
+  *ptr = NULL;
+  if (newSize < size)
+    return 12; // ENOMEM
+  p = MyAlloc(newSize);
+  if (!p)
+    return 12; // ENOMEM
+  pAligned = MY_ALIGN_PTR_UP_PLUS(p, align);
+  ((void **)pAligned)[-1] = p;
+  *ptr = pAligned;
+  return 0;
+}
+*/
+
+/*
+  ALLOC_ALIGN_SIZE >= sizeof(void *)
+  ALLOC_ALIGN_SIZE >= cache_line_size
+*/
+
+#define ALLOC_ALIGN_SIZE ((size_t)1 << 7)
+
+static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size)
+{
+  #ifndef USE_posix_memalign
+  
+  void *p;
+  void *pAligned;
+  size_t newSize;
+  UNUSED_VAR(pp);
+
+  /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned
+     block to prevent cache line sharing with another allocated blocks */
+
+  newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE;
+  if (newSize < size)
+    return NULL;
+
+  p = MyAlloc(newSize);
+  
+  if (!p)
+    return NULL;
+  pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE);
+
+  Print(" size="); PrintHex(size, 8);
+  Print(" a_size="); PrintHex(newSize, 8);
+  Print(" ptr="); PrintAddr(p);
+  Print(" a_ptr="); PrintAddr(pAligned);
+  PrintLn();
+
+  ((void **)pAligned)[-1] = p;
+
+  return pAligned;
+
+  #else
+
+  void *p;
+  UNUSED_VAR(pp);
+  if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size))
+    return NULL;
+
+  Print(" posix_memalign="); PrintAddr(p);
+  PrintLn();
+
+  return p;
+
+  #endif
+}
+
+
+static void SzAlignedFree(ISzAllocPtr pp, void *address)
+{
+  UNUSED_VAR(pp);
+  #ifndef USE_posix_memalign
+  if (address)
+    MyFree(((void **)address)[-1]);
+  #else
+  free(address);
+  #endif
+}
+
+
+const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree };
+
+
+
+#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *))
+
+/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */
+#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1]
+/*
+#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1]
+*/
+
+static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size)
+{
+  CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt);
+  void *adr;
+  void *pAligned;
+  size_t newSize;
+  size_t extra;
+  size_t alignSize = (size_t)1 << p->numAlignBits;
+
+  if (alignSize < sizeof(void *))
+    alignSize = sizeof(void *);
+  
+  if (p->offset >= alignSize)
+    return NULL;
+
+  /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned
+     block to prevent cache line sharing with another allocated blocks */
+  extra = p->offset & (sizeof(void *) - 1);
+  newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE;
+  if (newSize < size)
+    return NULL;
+
+  adr = ISzAlloc_Alloc(p->baseAlloc, newSize);
+  
+  if (!adr)
+    return NULL;
+
+  pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr +
+      alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset;
+
+  PrintLn();
+  Print("- Aligned: ");
+  Print(" size="); PrintHex(size, 8);
+  Print(" a_size="); PrintHex(newSize, 8);
+  Print(" ptr="); PrintAddr(adr);
+  Print(" a_ptr="); PrintAddr(pAligned);
+  PrintLn();
+
+  REAL_BLOCK_PTR_VAR(pAligned) = adr;
+
+  return pAligned;
+}
+
+
+static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address)
+{
+  if (address)
+  {
+    CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt);
+    PrintLn();
+    Print("- Aligned Free: ");
+    PrintLn();
+    ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address));
+  }
+}
+
+
+void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p)
+{
+  p->vt.Alloc = AlignOffsetAlloc_Alloc;
+  p->vt.Free = AlignOffsetAlloc_Free;
+}

C/Alloc.h

@@ -1,5 +1,5 @@
 /* Alloc.h -- Memory allocation functions
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-19 : Igor Pavlov : Public domain */
 
 #ifndef __COMMON_ALLOC_H
 #define __COMMON_ALLOC_H
@@ -31,6 +31,20 @@ void BigFree(void *address);
 
 extern const ISzAlloc g_Alloc;
 extern const ISzAlloc g_BigAlloc;
+extern const ISzAlloc g_MidAlloc;
+extern const ISzAlloc g_AlignedAlloc;
+
+
+typedef struct
+{
+  ISzAlloc vt;
+  ISzAllocPtr baseAlloc;
+  unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */
+  size_t offset;         /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */
+} CAlignOffsetAlloc;
+
+void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p);
+
 
 EXTERN_C_END
 

C/CpuArch.h

@@ -1,5 +1,5 @@
 /* CpuArch.h -- CPU specific code
-2017-04-03 : Igor Pavlov : Public domain */
+2017-09-04 : Igor Pavlov : Public domain */
 
 #ifndef __CPU_ARCH_H
 #define __CPU_ARCH_H
@@ -174,7 +174,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
 #ifndef MY_CPU_NAME
   #ifdef MY_CPU_LE
     #define MY_CPU_NAME "LE"
-  #elif MY_CPU_BE
+  #elif defined(MY_CPU_BE)
     #define MY_CPU_NAME "BE"
   #else
     /*
@@ -190,8 +190,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
 #ifdef MY_CPU_LE
   #if defined(MY_CPU_X86_OR_AMD64) \
       || defined(MY_CPU_ARM64) \
-      || defined(__ARM_FEATURE_UNALIGNED) \
-      || defined(__AARCH64EL__)
+      || defined(__ARM_FEATURE_UNALIGNED)
     #define MY_CPU_LE_UNALIGN
   #endif
 #endif
@@ -237,6 +236,11 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
 
 #endif
 
+#ifdef __has_builtin
+  #define MY__has_builtin(x) __has_builtin(x)
+#else
+  #define MY__has_builtin(x) 0
+#endif
 
 #if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300)
 
@@ -244,15 +248,21 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
 
 #include <stdlib.h>
 
+#pragma intrinsic(_byteswap_ushort)
 #pragma intrinsic(_byteswap_ulong)
 #pragma intrinsic(_byteswap_uint64)
+
+/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */
 #define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p))
 #define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p))
 
 #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v)
 
-#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
+#elif defined(MY_CPU_LE_UNALIGN) && ( \
+       (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \
+    || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) )
 
+/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */
 #define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p))
 #define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p))
 
@@ -277,10 +287,14 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
 #endif
 
 
+#ifndef GetBe16
+
 #define GetBe16(p) ( (UInt16) ( \
     ((UInt16)((const Byte *)(p))[0] << 8) | \
              ((const Byte *)(p))[1] ))
 
+#endif
+
 
 
 #ifdef MY_CPU_X86_OR_AMD64

C/LzFind.c

@@ -1,5 +1,5 @@
 /* LzFind.c -- Match finder for LZ algorithms
-2017-04-03 : Igor Pavlov : Public domain */
+2017-06-10 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -142,6 +142,7 @@ void MatchFinder_Construct(CMatchFinder *p)
   p->bufferBase = NULL;
   p->directInput = 0;
   p->hash = NULL;
+  p->expectedDataSize = (UInt64)(Int64)-1;
   MatchFinder_SetDefaultSettings(p);
 
   for (i = 0; i < 256; i++)
@@ -208,7 +209,11 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
         hs = (1 << 16) - 1;
       else
       {
-        hs = historySize - 1;
+        hs = historySize;
+        if (hs > p->expectedDataSize)
+          hs = (UInt32)p->expectedDataSize;
+        if (hs != 0)
+          hs--;
         hs |= (hs >> 1);
         hs |= (hs >> 2);
         hs |= (hs >> 4);
@@ -292,17 +297,33 @@ static void MatchFinder_SetLimits(CMatchFinder *p)
   p->posLimit = p->pos + limit;
 }
 
-void MatchFinder_Init_2(CMatchFinder *p, int readData)
+
+void MatchFinder_Init_LowHash(CMatchFinder *p)
+{
+  size_t i;
+  CLzRef *items = p->hash;
+  size_t numItems = p->fixedHashSize;
+  for (i = 0; i < numItems; i++)
+    items[i] = kEmptyHashValue;
+}
+
+
+void MatchFinder_Init_HighHash(CMatchFinder *p)
+{
+  size_t i;
+  CLzRef *items = p->hash + p->fixedHashSize;
+  size_t numItems = (size_t)p->hashMask + 1;
+  for (i = 0; i < numItems; i++)
+    items[i] = kEmptyHashValue;
+}
+
+
+void MatchFinder_Init_3(CMatchFinder *p, int readData)
 {
-  UInt32 i;
-  UInt32 *hash = p->hash;
-  UInt32 num = p->hashSizeSum;
-  for (i = 0; i < num; i++)
-    hash[i] = kEmptyHashValue;
-  
   p->cyclicBufferPos = 0;
   p->buffer = p->bufferBase;
-  p->pos = p->streamPos = p->cyclicBufferSize;
+  p->pos =
+  p->streamPos = p->cyclicBufferSize;
   p->result = SZ_OK;
   p->streamEndWasReached = 0;
   
@@ -312,10 +333,14 @@ void MatchFinder_Init_2(CMatchFinder *p, int readData)
   MatchFinder_SetLimits(p);
 }
 
+
 void MatchFinder_Init(CMatchFinder *p)
 {
-  MatchFinder_Init_2(p, True);
+  MatchFinder_Init_HighHash(p);
+  MatchFinder_Init_LowHash(p);
+  MatchFinder_Init_3(p, True);
 }
+
   
 static UInt32 MatchFinder_GetSubValue(CMatchFinder *p)
 {

C/LzFind.h

@@ -1,5 +1,5 @@
 /* LzFind.h -- Match finder for LZ algorithms
-2017-04-03 : Igor Pavlov : Public domain */
+2017-06-10 : Igor Pavlov : Public domain */
 
 #ifndef __LZ_FIND_H
 #define __LZ_FIND_H
@@ -47,6 +47,8 @@ typedef struct _CMatchFinder
   SRes result;
   UInt32 crc[256];
   size_t numRefs;
+
+  UInt64 expectedDataSize;
 } CMatchFinder;
 
 #define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)
@@ -103,7 +105,9 @@ typedef struct _IMatchFinder
 
 void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable);
 
-void MatchFinder_Init_2(CMatchFinder *p, int readData);
+void MatchFinder_Init_LowHash(CMatchFinder *p);
+void MatchFinder_Init_HighHash(CMatchFinder *p);
+void MatchFinder_Init_3(CMatchFinder *p, int readData);
 void MatchFinder_Init(CMatchFinder *p);
 
 UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);

C/LzFindMt.c

@@ -1,5 +1,5 @@
 /* LzFindMt.c -- multithreaded Match finder for LZ algorithms
-2017-04-03 : Igor Pavlov : Public domain */
+2017-06-10 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -33,6 +33,8 @@ static void MtSync_GetNextBlock(CMtSync *p)
 
     Event_Set(&p->canStart);
     Event_Wait(&p->wasStarted);
+
+    // if (mt) MatchFinder_Init_LowHash(mt->MatchFinder);
   }
   else
   {
@@ -155,6 +157,9 @@ static void HashThreadFunc(CMatchFinderMt *mt)
     UInt32 numProcessedBlocks = 0;
     Event_Wait(&p->canStart);
     Event_Set(&p->wasStarted);
+
+    MatchFinder_Init_HighHash(mt->MatchFinder);
+
     for (;;)
     {
       if (p->exit)
@@ -205,7 +210,7 @@ static void HashThreadFunc(CMatchFinderMt *mt)
             if (num > kMtHashBlockSize - 2)
               num = kMtHashBlockSize - 2;
             mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc);
-            heads[0] += num;
+            heads[0] = 2 + num;
           }
           mf->pos += num;
           mf->buffer += num;
@@ -496,14 +501,18 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB
 }
 
 /* Call it after ReleaseStream / SetStream */
-void MatchFinderMt_Init(CMatchFinderMt *p)
+static void MatchFinderMt_Init(CMatchFinderMt *p)
 {
   CMatchFinder *mf = p->MatchFinder;
-  p->btBufPos = p->btBufPosLimit = 0;
-  p->hashBufPos = p->hashBufPosLimit = 0;
+  
+  p->btBufPos =
+  p->btBufPosLimit = 0;
+  p->hashBufPos =
+  p->hashBufPosLimit = 0;
 
   /* Init without data reading. We don't want to read data in this thread */
-  MatchFinder_Init_2(mf, False);
+  MatchFinder_Init_3(mf, False);
+  MatchFinder_Init_LowHash(mf);
   
   p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf);
   p->btNumAvailBytes = 0;
@@ -684,8 +693,12 @@ static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances)
     UInt32 i;
     for (i = 0; i < len; i += 2)
     {
-      *distances++ = *btBuf++;
-      *distances++ = *btBuf++;
+      UInt32 v0 = btBuf[0];
+      UInt32 v1 = btBuf[1];
+      btBuf += 2;
+      distances[0] = v0;
+      distances[1] = v1;
+      distances += 2;
     }
   }
   INCREASE_LZ_POS
@@ -712,8 +725,12 @@ static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances)
     distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances);
     do
     {
-      *distances2++ = *btBuf++;
-      *distances2++ = *btBuf++;
+      UInt32 v0 = btBuf[0];
+      UInt32 v1 = btBuf[1];
+      btBuf += 2;
+      distances2[0] = v0;
+      distances2[1] = v1;
+      distances2 += 2;
     }
     while ((len -= 2) != 0);
     len = (UInt32)(distances2 - (distances));
@@ -777,7 +794,7 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable)
   {
     case 2:
       p->GetHeadsFunc = GetHeads2;
-      p->MixMatchesFunc = (Mf_Mix_Matches)0;
+      p->MixMatchesFunc = (Mf_Mix_Matches)NULL;
       vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip;
       vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches;
       break;

C/Lzma2Dec.c

@@ -1,5 +1,5 @@
 /* Lzma2Dec.c -- LZMA2 Decoder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-19 : Igor Pavlov : Public domain */
 
 /* #define SHOW_DEBUG_INFO */
 
@@ -14,28 +14,22 @@
 #include "Lzma2Dec.h"
 
 /*
-00000000  -  EOS
-00000001 U U  -  Uncompressed Reset Dic
-00000010 U U  -  Uncompressed No Reset
-100uuuuu U U P P  -  LZMA no reset
-101uuuuu U U P P  -  LZMA reset state
-110uuuuu U U P P S  -  LZMA reset state + new prop
-111uuuuu U U P P S  -  LZMA reset state + new prop + reset dic
+00000000  -  End of data
+00000001 U U  -  Uncompressed, reset dic, need reset state and set new prop
+00000010 U U  -  Uncompressed, no reset
+100uuuuu U U P P  -  LZMA, no reset
+101uuuuu U U P P  -  LZMA, reset state
+110uuuuu U U P P S  -  LZMA, reset state + set new prop
+111uuuuu U U P P S  -  LZMA, reset state + set new prop, reset dic
 
   u, U - Unpack Size
   P - Pack Size
   S - Props
 */
 
-#define LZMA2_CONTROL_LZMA (1 << 7)
-#define LZMA2_CONTROL_COPY_NO_RESET 2
 #define LZMA2_CONTROL_COPY_RESET_DIC 1
-#define LZMA2_CONTROL_EOF 0
 
-#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0)
-
-#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3)
-#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2)
+#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0)
 
 #define LZMA2_LCLP_MAX 4
 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))
@@ -91,9 +85,11 @@ SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc)
 void Lzma2Dec_Init(CLzma2Dec *p)
 {
   p->state = LZMA2_STATE_CONTROL;
-  p->needInitDic = True;
-  p->needInitState = True;
-  p->needInitProp = True;
+  p->needInitLevel = 0xE0;
+  p->isExtraMode = False;
+  p->unpackSize = 0;
+  
+  // p->decoder.dicPos = 0; // we can use it instead of full init
   LzmaDec_Init(&p->decoder);
 }
 
@@ -102,19 +98,26 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b)
   switch (p->state)
   {
     case LZMA2_STATE_CONTROL:
+      p->isExtraMode = False;
       p->control = b;
-      PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos));
-      PRF(printf(" %2X", (unsigned)b));
+      PRF(printf("\n %8X", (unsigned)p->decoder.dicPos));
+      PRF(printf(" %02X", (unsigned)b));
       if (b == 0)
         return LZMA2_STATE_FINISHED;
       if (LZMA2_IS_UNCOMPRESSED_STATE(p))
       {
-        if (b > 2)
+        if (b == LZMA2_CONTROL_COPY_RESET_DIC)
+          p->needInitLevel = 0xC0;
+        else if (b > 2 || p->needInitLevel == 0xE0)
           return LZMA2_STATE_ERROR;
-        p->unpackSize = 0;
       }
       else
+      {
+        if (b < p->needInitLevel)
+          return LZMA2_STATE_ERROR;
+        p->needInitLevel = 0;
         p->unpackSize = (UInt32)(b & 0x1F) << 16;
+      }
       return LZMA2_STATE_UNPACK0;
     
     case LZMA2_STATE_UNPACK0:
@@ -124,8 +127,8 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b)
     case LZMA2_STATE_UNPACK1:
       p->unpackSize |= (UInt32)b;
       p->unpackSize++;
-      PRF(printf(" %8u", (unsigned)p->unpackSize));
-      return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0;
+      PRF(printf(" %7u", (unsigned)p->unpackSize));
+      return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0;
     
     case LZMA2_STATE_PACK0:
       p->packSize = (UInt32)b << 8;
@@ -134,9 +137,9 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b)
     case LZMA2_STATE_PACK1:
       p->packSize |= (UInt32)b;
       p->packSize++;
-      PRF(printf(" %8u", (unsigned)p->packSize));
-      return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP:
-        (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA);
+      // if (p->packSize < 5) return LZMA2_STATE_ERROR;
+      PRF(printf(" %5u", (unsigned)p->packSize));
+      return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA;
 
     case LZMA2_STATE_PROP:
     {
@@ -145,13 +148,12 @@ static ELzma2State Lzma2Dec_UpdateState(CLzma2Dec *p, Byte b)
         return LZMA2_STATE_ERROR;
       lc = b % 9;
       b /= 9;
-      p->decoder.prop.pb = b / 5;
+      p->decoder.prop.pb = (Byte)(b / 5);
       lp = b % 5;
       if (lc + lp > LZMA2_LCLP_MAX)
         return LZMA2_STATE_ERROR;
-      p->decoder.prop.lc = lc;
-      p->decoder.prop.lp = lp;
-      p->needInitProp = False;
+      p->decoder.prop.lc = (Byte)lc;
+      p->decoder.prop.lp = (Byte)lp;
       return LZMA2_STATE_DATA;
     }
   }
@@ -231,11 +233,6 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
         if (p->state == LZMA2_STATE_DATA)
         {
           Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC);
-          if (initDic)
-            p->needInitProp = p->needInitState = True;
-          else if (p->needInitDic)
-            break;
-          p->needInitDic = False;
           LzmaDec_InitDicAndState(&p->decoder, initDic, False);
         }
 
@@ -257,23 +254,17 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
 
         if (p->state == LZMA2_STATE_DATA)
         {
-          unsigned mode = LZMA2_GET_LZMA_MODE(p);
-          Bool initDic = (mode == 3);
-          Bool initState = (mode != 0);
-          if ((!initDic && p->needInitDic) || (!initState && p->needInitState))
-            break;
-
+          Bool initDic = (p->control >= 0xE0);
+          Bool initState = (p->control >= 0xA0);
           LzmaDec_InitDicAndState(&p->decoder, initDic, initState);
-          p->needInitDic = False;
-          p->needInitState = False;
           p->state = LZMA2_STATE_DATA_CONT;
         }
   
         if (inCur > p->packSize)
           inCur = (SizeT)p->packSize;
-          
-        res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status);
         
+        res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status);
+
         src += inCur;
         *srcLen += inCur;
         p->packSize -= (UInt32)inCur;
@@ -310,6 +301,129 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
 }
 
 
+
+
+ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
+    SizeT outSize,
+    const Byte *src, SizeT *srcLen,
+    int checkFinishBlock)
+{
+  SizeT inSize = *srcLen;
+  *srcLen = 0;
+
+  while (p->state != LZMA2_STATE_ERROR)
+  {
+    if (p->state == LZMA2_STATE_FINISHED)
+      return LZMA_STATUS_FINISHED_WITH_MARK;
+
+    if (outSize == 0 && !checkFinishBlock)
+      return LZMA_STATUS_NOT_FINISHED;
+    
+    if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT)
+    {
+      if (*srcLen == inSize)
+        return LZMA_STATUS_NEEDS_MORE_INPUT;
+      (*srcLen)++;
+
+      p->state = Lzma2Dec_UpdateState(p, *src++);
+
+      if (p->state == LZMA2_STATE_UNPACK0)
+      {
+        // if (p->decoder.dicPos != 0)
+        if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0)
+          return LZMA2_PARSE_STATUS_NEW_BLOCK;
+        // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED;
+      }
+
+      // The following code can be commented.
+      // It's not big problem, if we read additional input bytes.
+      // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state.
+
+      if (outSize == 0 && p->state != LZMA2_STATE_FINISHED)
+      {
+        // checkFinishBlock is true. So we expect that block must be finished,
+        // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here
+        // break;
+        return LZMA_STATUS_NOT_FINISHED;
+      }
+
+      if (p->state == LZMA2_STATE_DATA)
+        return LZMA2_PARSE_STATUS_NEW_CHUNK;
+
+      continue;
+    }
+
+    if (outSize == 0)
+      return LZMA_STATUS_NOT_FINISHED;
+
+    {
+      SizeT inCur = inSize - *srcLen;
+
+      if (LZMA2_IS_UNCOMPRESSED_STATE(p))
+      {
+        if (inCur == 0)
+          return LZMA_STATUS_NEEDS_MORE_INPUT;
+        if (inCur > p->unpackSize)
+          inCur = p->unpackSize;
+        if (inCur > outSize)
+          inCur = outSize;
+        p->decoder.dicPos += inCur;
+        src += inCur;
+        *srcLen += inCur;
+        outSize -= inCur;
+        p->unpackSize -= (UInt32)inCur;
+        p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT;
+      }
+      else
+      {
+        p->isExtraMode = True;
+
+        if (inCur == 0)
+        {
+          if (p->packSize != 0)
+            return LZMA_STATUS_NEEDS_MORE_INPUT;
+        }
+        else if (p->state == LZMA2_STATE_DATA)
+        {
+          p->state = LZMA2_STATE_DATA_CONT;
+          if (*src != 0)
+          {
+            // first byte of lzma chunk must be Zero
+            *srcLen += 1;
+            p->packSize--;
+            break;
+          }
+        }
+  
+        if (inCur > p->packSize)
+          inCur = (SizeT)p->packSize;
+
+        src += inCur;
+        *srcLen += inCur;
+        p->packSize -= (UInt32)inCur;
+
+        if (p->packSize == 0)
+        {
+          SizeT rem = outSize;
+          if (rem > p->unpackSize)
+            rem = p->unpackSize;
+          p->decoder.dicPos += rem;
+          p->unpackSize -= (UInt32)rem;
+          outSize -= rem;
+          if (p->unpackSize == 0)
+            p->state = LZMA2_STATE_CONTROL;
+        }
+      }
+    }
+  }
+  
+  p->state = LZMA2_STATE_ERROR;
+  return LZMA_STATUS_NOT_SPECIFIED;
+}
+
+
+
+
 SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status)
 {
   SizeT outSize = *destLen, inSize = *srcLen;

C/Lzma2Dec.h

@@ -1,5 +1,5 @@
 /* Lzma2Dec.h -- LZMA2 Decoder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-19 : Igor Pavlov : Public domain */
 
 #ifndef __LZMA2_DEC_H
 #define __LZMA2_DEC_H
@@ -12,25 +12,24 @@ EXTERN_C_BEGIN
 
 typedef struct
 {
-  CLzmaDec decoder;
-  UInt32 packSize;
-  UInt32 unpackSize;
   unsigned state;
   Byte control;
-  Bool needInitDic;
-  Bool needInitState;
-  Bool needInitProp;
+  Byte needInitLevel;
+  Byte isExtraMode;
+  Byte _pad_;
+  UInt32 packSize;
+  UInt32 unpackSize;
+  CLzmaDec decoder;
 } CLzma2Dec;
 
 #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder)
-#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc);
-#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc);
+#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc)
+#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc)
 
 SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc);
 SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc);
 void Lzma2Dec_Init(CLzma2Dec *p);
 
-
 /*
 finishMode:
   It has meaning only if the decoding reaches output limit (*destLen or dicLimit).
@@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen,
     const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status);
 
 
+/* ---------- LZMA2 block and chunk parsing ---------- */
+
+/*
+Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data.
+It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code:
+  - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input.
+  - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read.
+                                   CLzma2Dec::unpackSize contains unpack size of that chunk
+*/
+
+typedef enum
+{
+/*
+  LZMA_STATUS_NOT_SPECIFIED                 // data error
+  LZMA_STATUS_FINISHED_WITH_MARK
+  LZMA_STATUS_NOT_FINISHED                  //
+  LZMA_STATUS_NEEDS_MORE_INPUT
+  LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK   // unused
+*/
+  LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1,
+  LZMA2_PARSE_STATUS_NEW_CHUNK
+} ELzma2ParseStatus;
+
+ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
+    SizeT outSize,   // output size
+    const Byte *src, SizeT *srcLen,
+    int checkFinishBlock   // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position.
+    );
+
+/*
+LZMA2 parser doesn't decode LZMA chunks, so we must read
+  full input LZMA chunk to decode some part of LZMA chunk.
+
+Lzma2Dec_GetUnpackExtra() returns the value that shows
+    max possible number of output bytes that can be output by decoder
+    at current input positon.
+*/
+
+#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0);
+
+
 /* ---------- One Call Interface ---------- */
 
 /*

C/Lzma2DecMt.h

@@ -0,0 +1,79 @@
+/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread
+2018-02-17 : Igor Pavlov : Public domain */
+
+#ifndef __LZMA2_DEC_MT_H
+#define __LZMA2_DEC_MT_H
+
+#include "7zTypes.h"
+
+EXTERN_C_BEGIN
+
+typedef struct
+{
+  size_t inBufSize_ST;
+  size_t outStep_ST;
+  
+  #ifndef _7ZIP_ST
+  unsigned numThreads;
+  size_t inBufSize_MT;
+  size_t outBlockMax;
+  size_t inBlockMax;
+  #endif
+} CLzma2DecMtProps;
+
+/* init to single-thread mode */
+void Lzma2DecMtProps_Init(CLzma2DecMtProps *p);
+
+
+/* ---------- CLzma2DecMtHandle Interface ---------- */
+
+/* Lzma2DecMt_ * functions can return the following exit codes:
+SRes:
+  SZ_OK           - OK
+  SZ_ERROR_MEM    - Memory allocation error
+  SZ_ERROR_PARAM  - Incorrect paramater in props
+  SZ_ERROR_WRITE  - ISeqOutStream write callback error
+  // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
+  SZ_ERROR_PROGRESS - some break from progress callback
+  SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
+*/
+
+typedef void * CLzma2DecMtHandle;
+
+CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid);
+void Lzma2DecMt_Destroy(CLzma2DecMtHandle p);
+
+SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p,
+    Byte prop,
+    const CLzma2DecMtProps *props,
+    ISeqOutStream *outStream,
+    const UInt64 *outDataSize, // NULL means undefined
+    int finishMode,            // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished
+    // Byte *outBuf, size_t *outBufSize,
+    ISeqInStream *inStream,
+    // const Byte *inData, size_t inDataSize,
+    
+    // out variables:
+    UInt64 *inProcessed,
+    int *isMT,  /* out: (*isMT == 0), if single thread decoding was used */
+
+    // UInt64 *outProcessed,
+    ICompressProgress *progress);
+
+
+/* ---------- Read from CLzma2DecMtHandle Interface ---------- */
+
+SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp,
+    Byte prop,
+    const CLzma2DecMtProps *props,
+    const UInt64 *outDataSize, int finishMode,
+    ISeqInStream *inStream);
+
+SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp,
+    Byte *data, size_t *outSize,
+    UInt64 *inStreamProcessed);
+
+
+EXTERN_C_END
+
+#endif

C/Lzma2Enc.c

@@ -1,9 +1,8 @@
 /* Lzma2Enc.c -- LZMA2 Encoder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-08 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
-/* #include <stdio.h> */
 #include <string.h>
 
 /* #define _7ZIP_ST */
@@ -13,7 +12,7 @@
 #ifndef _7ZIP_ST
 #include "MtCoder.h"
 #else
-#define NUM_MT_CODER_THREADS_MAX 1
+#define MTCODER__THREADS_MAX 1
 #endif
 
 #define LZMA2_CONTROL_LZMA (1 << 7)
@@ -35,30 +34,83 @@
 
 #define PRF(x) /* x */
 
+
+/* ---------- CLimitedSeqInStream ---------- */
+
+typedef struct
+{
+  ISeqInStream vt;
+  ISeqInStream *realStream;
+  UInt64 limit;
+  UInt64 processed;
+  int finished;
+} CLimitedSeqInStream;
+
+static void LimitedSeqInStream_Init(CLimitedSeqInStream *p)
+{
+  p->limit = (UInt64)(Int64)-1;
+  p->processed = 0;
+  p->finished = 0;
+}
+
+static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size)
+{
+  CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt);
+  size_t size2 = *size;
+  SRes res = SZ_OK;
+  
+  if (p->limit != (UInt64)(Int64)-1)
+  {
+    UInt64 rem = p->limit - p->processed;
+    if (size2 > rem)
+      size2 = (size_t)rem;
+  }
+  if (size2 != 0)
+  {
+    res = ISeqInStream_Read(p->realStream, data, &size2);
+    p->finished = (size2 == 0 ? 1 : 0);
+    p->processed += size2;
+  }
+  *size = size2;
+  return res;
+}
+
+
 /* ---------- CLzma2EncInt ---------- */
 
 typedef struct
 {
   CLzmaEncHandle enc;
+  Byte propsAreSet;
+  Byte propsByte;
+  Byte needInitState;
+  Byte needInitProp;
   UInt64 srcPos;
-  Byte props;
-  Bool needInitState;
-  Bool needInitProp;
 } CLzma2EncInt;
 
-static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
+
+static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props)
 {
-  Byte propsEncoded[LZMA_PROPS_SIZE];
-  SizeT propsSize = LZMA_PROPS_SIZE;
-  RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
-  RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
-  p->srcPos = 0;
-  p->props = propsEncoded[0];
-  p->needInitState = True;
-  p->needInitProp = True;
+  if (!p->propsAreSet)
+  {
+    SizeT propsSize = LZMA_PROPS_SIZE;
+    Byte propsEncoded[LZMA_PROPS_SIZE];
+    RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
+    RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
+    p->propsByte = propsEncoded[0];
+    p->propsAreSet = True;
+  }
   return SZ_OK;
 }
 
+static void Lzma2EncInt_InitBlock(CLzma2EncInt *p)
+{
+  p->srcPos = 0;
+  p->needInitState = True;
+  p->needInitProp = True;
+}
+
+
 SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
     ISzAllocPtr alloc, ISzAllocPtr allocBig);
 SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
@@ -70,6 +122,9 @@ void LzmaEnc_Finish(CLzmaEncHandle pp);
 void LzmaEnc_SaveState(CLzmaEncHandle pp);
 void LzmaEnc_RestoreState(CLzmaEncHandle pp);
 
+/*
+UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp);
+*/
 
 static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
     size_t *packSizeRes, ISeqOutStream *outStream)
@@ -154,7 +209,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
     outBuf[destPos++] = (Byte)pm;
     
     if (p->needInitProp)
-      outBuf[destPos++] = p->props;
+      outBuf[destPos++] = p->propsByte;
     
     p->needInitProp = False;
     p->needInitState = False;
@@ -176,14 +231,16 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
 void Lzma2EncProps_Init(CLzma2EncProps *p)
 {
   LzmaEncProps_Init(&p->lzmaProps);
+  p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO;
+  p->numBlockThreads_Reduced = -1;
+  p->numBlockThreads_Max = -1;
   p->numTotalThreads = -1;
-  p->numBlockThreads = -1;
-  p->blockSize = 0;
 }
 
 void Lzma2EncProps_Normalize(CLzma2EncProps *p)
 {
-  int t1, t1n, t2, t3;
+  UInt64 fileSize;
+  int t1, t1n, t2, t2r, t3;
   {
     CLzmaEncProps lzmaProps = p->lzmaProps;
     LzmaEncProps_Normalize(&lzmaProps);
@@ -191,11 +248,11 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
   }
 
   t1 = p->lzmaProps.numThreads;
-  t2 = p->numBlockThreads;
+  t2 = p->numBlockThreads_Max;
   t3 = p->numTotalThreads;
 
-  if (t2 > NUM_MT_CODER_THREADS_MAX)
-    t2 = NUM_MT_CODER_THREADS_MAX;
+  if (t2 > MTCODER__THREADS_MAX)
+    t2 = MTCODER__THREADS_MAX;
 
   if (t3 <= 0)
   {
@@ -211,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
       t1 = 1;
       t2 = t3;
     }
-    if (t2 > NUM_MT_CODER_THREADS_MAX)
-      t2 = NUM_MT_CODER_THREADS_MAX;
+    if (t2 > MTCODER__THREADS_MAX)
+      t2 = MTCODER__THREADS_MAX;
   }
   else if (t1 <= 0)
   {
@@ -225,39 +282,64 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
 
   p->lzmaProps.numThreads = t1;
 
+  t2r = t2;
+
+  fileSize = p->lzmaProps.reduceSize;
+
+  if (   p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
+      && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
+      && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1))
+    p->lzmaProps.reduceSize = p->blockSize;
+
   LzmaEncProps_Normalize(&p->lzmaProps);
 
+  p->lzmaProps.reduceSize = fileSize;
+
   t1 = p->lzmaProps.numThreads;
 
-  if (p->blockSize == 0)
+  if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID)
   {
-    UInt32 dictSize = p->lzmaProps.dictSize;
-    UInt64 blockSize = (UInt64)dictSize << 2;
-    const UInt32 kMinSize = (UInt32)1 << 20;
-    const UInt32 kMaxSize = (UInt32)1 << 28;
-    if (blockSize < kMinSize) blockSize = kMinSize;
-    if (blockSize > kMaxSize) blockSize = kMaxSize;
-    if (blockSize < dictSize) blockSize = dictSize;
-    p->blockSize = (size_t)blockSize;
+    t2r = t2 = 1;
+    t3 = t1;
   }
-  
-  if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)
+  else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1)
   {
-    UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
-    if (temp > p->lzmaProps.reduceSize)
+    /* if there is no block multi-threading, we use SOLID block */
+    p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID;
+  }
+  else
+  {
+    if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO)
     {
-      UInt64 numBlocks = temp / p->blockSize;
+      const UInt32 kMinSize = (UInt32)1 << 20;
+      const UInt32 kMaxSize = (UInt32)1 << 28;
+      const UInt32 dictSize = p->lzmaProps.dictSize;
+      UInt64 blockSize = (UInt64)dictSize << 2;
+      if (blockSize < kMinSize) blockSize = kMinSize;
+      if (blockSize > kMaxSize) blockSize = kMaxSize;
+      if (blockSize < dictSize) blockSize = dictSize;
+      blockSize += (kMinSize - 1);
+      blockSize &= ~(UInt64)(kMinSize - 1);
+      p->blockSize = blockSize;
+    }
+    
+    if (t2 > 1 && fileSize != (UInt64)(Int64)-1)
+    {
+      UInt64 numBlocks = fileSize / p->blockSize;
+      if (numBlocks * p->blockSize != fileSize)
+        numBlocks++;
       if (numBlocks < (unsigned)t2)
       {
-        t2 = (unsigned)numBlocks;
-        if (t2 == 0)
-          t2 = 1;
-        t3 = t1 * t2;
+        t2r = (unsigned)numBlocks;
+        if (t2r == 0)
+          t2r = 1;
+        t3 = t1 * t2r;
       }
     }
   }
   
-  p->numBlockThreads = t2;
+  p->numBlockThreads_Max = t2;
+  p->numBlockThreads_Reduced = t2r;
   p->numTotalThreads = t3;
 }
 
@@ -274,134 +356,30 @@ typedef struct
 {
   Byte propEncoded;
   CLzma2EncProps props;
+  UInt64 expectedDataSize;
   
-  Byte *outBuf;
+  Byte *tempBufLzma;
 
   ISzAllocPtr alloc;
   ISzAllocPtr allocBig;
 
-  CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
+  CLzma2EncInt coders[MTCODER__THREADS_MAX];
 
   #ifndef _7ZIP_ST
+  
+  ISeqOutStream *outStream;
+  Byte *outBuf;
+  size_t outBufSize;
+  size_t outBufsDataSizes[MTCODER__BLOCKS_MAX];
+  Bool mtCoder_WasConstructed;
   CMtCoder mtCoder;
+  Byte *outBufs[MTCODER__BLOCKS_MAX];
+
   #endif
 
 } CLzma2Enc;
 
 
-/* ---------- Lzma2EncThread ---------- */
-
-static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
-  ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
-{
-  UInt64 packTotal = 0;
-  SRes res = SZ_OK;
-
-  if (!mainEncoder->outBuf)
-  {
-    mainEncoder->outBuf = (Byte *)ISzAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
-    if (!mainEncoder->outBuf)
-      return SZ_ERROR_MEM;
-  }
-  
-  RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
-  RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
-      mainEncoder->alloc, mainEncoder->allocBig));
-  
-  for (;;)
-  {
-    size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
-    res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
-    if (res != SZ_OK)
-      break;
-    packTotal += packSize;
-    res = Progress(progress, p->srcPos, packTotal);
-    if (res != SZ_OK)
-      break;
-    if (packSize == 0)
-      break;
-  }
-  
-  LzmaEnc_Finish(p->enc);
-
-  if (res == SZ_OK)
-  {
-    Byte b = 0;
-    if (ISeqOutStream_Write(outStream, &b, 1) != 1)
-      return SZ_ERROR_WRITE;
-  }
-  
-  return res;
-}
-
-
-#ifndef _7ZIP_ST
-
-typedef struct
-{
-  IMtCoderCallback funcTable;
-  CLzma2Enc *lzma2Enc;
-} CMtCallbackImp;
-
-static SRes MtCallbackImp_Code(const IMtCoderCallback *pp, unsigned index, Byte *dest, size_t *destSize,
-      const Byte *src, size_t srcSize, int finished)
-{
-  CMtCallbackImp *imp = CONTAINER_FROM_VTBL(pp, CMtCallbackImp, funcTable);
-  CLzma2Enc *mainEncoder = imp->lzma2Enc;
-  CLzma2EncInt *p = &mainEncoder->coders[index];
-
-  SRes res = SZ_OK;
-  {
-    size_t destLim = *destSize;
-    *destSize = 0;
-
-    if (srcSize != 0)
-    {
-      RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
-     
-      RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
-          mainEncoder->alloc, mainEncoder->allocBig));
-     
-      while (p->srcPos < srcSize)
-      {
-        size_t packSize = destLim - *destSize;
-        res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
-        if (res != SZ_OK)
-          break;
-        *destSize += packSize;
-
-        if (packSize == 0)
-        {
-          res = SZ_ERROR_FAIL;
-          break;
-        }
-
-        if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
-        {
-          res = SZ_ERROR_PROGRESS;
-          break;
-        }
-      }
-      
-      LzmaEnc_Finish(p->enc);
-      if (res != SZ_OK)
-        return res;
-    }
-    
-    if (finished)
-    {
-      if (*destSize == destLim)
-        return SZ_ERROR_OUTPUT_EOF;
-      dest[(*destSize)++] = 0;
-    }
-  }
-  return res;
-}
-
-#endif
-
-
-/* ---------- Lzma2Enc ---------- */
 
 CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)
 {
@@ -410,44 +388,78 @@ CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)
     return NULL;
   Lzma2EncProps_Init(&p->props);
   Lzma2EncProps_Normalize(&p->props);
-  p->outBuf = 0;
+  p->expectedDataSize = (UInt64)(Int64)-1;
+  p->tempBufLzma = NULL;
   p->alloc = alloc;
   p->allocBig = allocBig;
   {
     unsigned i;
-    for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
-      p->coders[i].enc = 0;
+    for (i = 0; i < MTCODER__THREADS_MAX; i++)
+      p->coders[i].enc = NULL;
   }
   
   #ifndef _7ZIP_ST
-  MtCoder_Construct(&p->mtCoder);
+  p->mtCoder_WasConstructed = False;
+  {
+    unsigned i;
+    for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
+      p->outBufs[i] = NULL;
+    p->outBufSize = 0;
+  }
   #endif
 
   return p;
 }
 
+
+#ifndef _7ZIP_ST
+
+static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p)
+{
+  unsigned i;
+  for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
+    if (p->outBufs[i])
+    {
+      ISzAlloc_Free(p->alloc, p->outBufs[i]);
+      p->outBufs[i] = NULL;
+    }
+  p->outBufSize = 0;
+}
+
+#endif
+
+
 void Lzma2Enc_Destroy(CLzma2EncHandle pp)
 {
   CLzma2Enc *p = (CLzma2Enc *)pp;
   unsigned i;
-  for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
+  for (i = 0; i < MTCODER__THREADS_MAX; i++)
   {
     CLzma2EncInt *t = &p->coders[i];
     if (t->enc)
     {
       LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
-      t->enc = 0;
+      t->enc = NULL;
     }
   }
 
+
   #ifndef _7ZIP_ST
-  MtCoder_Destruct(&p->mtCoder);
+  if (p->mtCoder_WasConstructed)
+  {
+    MtCoder_Destruct(&p->mtCoder);
+    p->mtCoder_WasConstructed = False;
+  }
+  Lzma2Enc_FreeOutBufs(p);
   #endif
 
-  ISzAlloc_Free(p->alloc, p->outBuf);
+  ISzAlloc_Free(p->alloc, p->tempBufLzma);
+  p->tempBufLzma = NULL;
+
   ISzAlloc_Free(p->alloc, pp);
 }
 
+
 SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
 {
   CLzma2Enc *p = (CLzma2Enc *)pp;
@@ -460,6 +472,14 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
   return SZ_OK;
 }
 
+
+void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize)
+{
+  CLzma2Enc *p = (CLzma2Enc *)pp;
+  p->expectedDataSize = expectedDataSiize;
+}
+
+
 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
 {
   CLzma2Enc *p = (CLzma2Enc *)pp;
@@ -471,50 +491,311 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
   return (Byte)i;
 }
 
-SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
-    ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
-{
-  CLzma2Enc *p = (CLzma2Enc *)pp;
-  int i;
 
-  for (i = 0; i < p->props.numBlockThreads; i++)
+static SRes Lzma2Enc_EncodeMt1(
+    CLzma2Enc *me,
+    CLzma2EncInt *p,
+    ISeqOutStream *outStream,
+    Byte *outBuf, size_t *outBufSize,
+    ISeqInStream *inStream,
+    const Byte *inData, size_t inDataSize,
+    int finished,
+    ICompressProgress *progress)
+{
+  UInt64 unpackTotal = 0;
+  UInt64 packTotal = 0;
+  size_t outLim = 0;
+  CLimitedSeqInStream limitedInStream;
+
+  if (outBuf)
   {
-    CLzma2EncInt *t = &p->coders[(unsigned)i];
-    if (!t->enc)
+    outLim = *outBufSize;
+    *outBufSize = 0;
+  }
+
+  if (!p->enc)
+  {
+    p->propsAreSet = False;
+    p->enc = LzmaEnc_Create(me->alloc);
+    if (!p->enc)
+      return SZ_ERROR_MEM;
+  }
+
+  limitedInStream.realStream = inStream;
+  if (inStream)
+  {
+    limitedInStream.vt.Read = LimitedSeqInStream_Read;
+  }
+  
+  if (!outBuf)
+  {
+    // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma
+    if (!me->tempBufLzma)
     {
-      t->enc = LzmaEnc_Create(p->alloc);
-      if (!t->enc)
+      me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
+      if (!me->tempBufLzma)
         return SZ_ERROR_MEM;
     }
   }
 
-  #ifndef _7ZIP_ST
-  if (p->props.numBlockThreads > 1)
-  {
-    CMtCallbackImp mtCallback;
+  RINOK(Lzma2EncInt_InitStream(p, &me->props));
 
-    mtCallback.funcTable.Code = MtCallbackImp_Code;
-    mtCallback.lzma2Enc = p;
+  for (;;)
+  {
+    SRes res = SZ_OK;
+    size_t inSizeCur = 0;
+
+    Lzma2EncInt_InitBlock(p);
     
+    LimitedSeqInStream_Init(&limitedInStream);
+    limitedInStream.limit = me->props.blockSize;
+
+    if (inStream)
+    {
+      UInt64 expected = (UInt64)(Int64)-1;
+      // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize
+      if (me->expectedDataSize != (UInt64)(Int64)-1
+          && me->expectedDataSize >= unpackTotal)
+        expected = me->expectedDataSize - unpackTotal;
+      if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
+          && expected > me->props.blockSize)
+        expected = (size_t)me->props.blockSize;
+
+      LzmaEnc_SetDataSize(p->enc, expected);
+
+      RINOK(LzmaEnc_PrepareForLzma2(p->enc,
+          &limitedInStream.vt,
+          LZMA2_KEEP_WINDOW_SIZE,
+          me->alloc,
+          me->allocBig));
+    }
+    else
+    {
+      inSizeCur = inDataSize - (size_t)unpackTotal;
+      if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
+          && inSizeCur > me->props.blockSize)
+        inSizeCur = (size_t)me->props.blockSize;
+    
+      // LzmaEnc_SetDataSize(p->enc, inSizeCur);
+      
+      RINOK(LzmaEnc_MemPrepare(p->enc,
+          inData + (size_t)unpackTotal, inSizeCur,
+          LZMA2_KEEP_WINDOW_SIZE,
+          me->alloc,
+          me->allocBig));
+    }
+
+    for (;;)
+    {
+      size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
+      if (outBuf)
+        packSize = outLim - (size_t)packTotal;
+      
+      res = Lzma2EncInt_EncodeSubblock(p,
+          outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize,
+          outBuf ? NULL : outStream);
+      
+      if (res != SZ_OK)
+        break;
+
+      packTotal += packSize;
+      if (outBuf)
+        *outBufSize = (size_t)packTotal;
+      
+      res = Progress(progress, unpackTotal + p->srcPos, packTotal);
+      if (res != SZ_OK)
+        break;
+
+      /*
+      if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0)
+        break;
+      */
+
+      if (packSize == 0)
+        break;
+    }
+    
+    LzmaEnc_Finish(p->enc);
+    
+    unpackTotal += p->srcPos;
+    
+    RINOK(res);
+
+    if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur))
+      return SZ_ERROR_FAIL;
+    
+    if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize))
+    {
+      if (finished)
+      {
+        if (outBuf)
+        {
+          size_t destPos = *outBufSize;
+          if (destPos >= outLim)
+            return SZ_ERROR_OUTPUT_EOF;
+          outBuf[destPos] = 0;
+          *outBufSize = destPos + 1;
+        }
+        else
+        {
+          Byte b = 0;
+          if (ISeqOutStream_Write(outStream, &b, 1) != 1)
+            return SZ_ERROR_WRITE;
+        }
+      }
+      return SZ_OK;
+    }
+  }
+}
+
+
+
+#ifndef _7ZIP_ST
+
+static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex,
+    const Byte *src, size_t srcSize, int finished)
+{
+  CLzma2Enc *me = (CLzma2Enc *)pp;
+  size_t destSize = me->outBufSize;
+  SRes res;
+  CMtProgressThunk progressThunk;
+
+  Byte *dest = me->outBufs[outBufIndex];
+
+  me->outBufsDataSizes[outBufIndex] = 0;
+
+  if (!dest)
+  {
+    dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize);
+    if (!dest)
+      return SZ_ERROR_MEM;
+    me->outBufs[outBufIndex] = dest;
+  }
+
+  MtProgressThunk_CreateVTable(&progressThunk);
+  progressThunk.mtProgress = &me->mtCoder.mtProgress;
+  progressThunk.inSize = 0;
+  progressThunk.outSize = 0;
+
+  res = Lzma2Enc_EncodeMt1(me,
+      &me->coders[coderIndex],
+      NULL, dest, &destSize,
+      NULL, src, srcSize,
+      finished,
+      &progressThunk.vt);
+
+  me->outBufsDataSizes[outBufIndex] = destSize;
+
+  return res;
+}
+
+
+static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex)
+{
+  CLzma2Enc *me = (CLzma2Enc *)pp;
+  size_t size = me->outBufsDataSizes[outBufIndex];
+  const Byte *data = me->outBufs[outBufIndex];
+  
+  if (me->outStream)
+    return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE;
+  
+  if (size > me->outBufSize)
+    return SZ_ERROR_OUTPUT_EOF;
+  memcpy(me->outBuf, data, size);
+  me->outBufSize -= size;
+  me->outBuf += size;
+  return SZ_OK;
+}
+
+#endif
+
+
+
+SRes Lzma2Enc_Encode2(CLzma2EncHandle pp,
+    ISeqOutStream *outStream,
+    Byte *outBuf, size_t *outBufSize,
+    ISeqInStream *inStream,
+    const Byte *inData, size_t inDataSize,
+    ICompressProgress *progress)
+{
+  CLzma2Enc *p = (CLzma2Enc *)pp;
+
+  if (inStream && inData)
+    return SZ_ERROR_PARAM;
+
+  if (outStream && outBuf)
+    return SZ_ERROR_PARAM;
+
+  {
+    unsigned i;
+    for (i = 0; i < MTCODER__THREADS_MAX; i++)
+      p->coders[i].propsAreSet = False;
+  }
+
+  #ifndef _7ZIP_ST
+  
+  if (p->props.numBlockThreads_Reduced > 1)
+  {
+    IMtCoderCallback2 vt;
+
+    if (!p->mtCoder_WasConstructed)
+    {
+      p->mtCoder_WasConstructed = True;
+      MtCoder_Construct(&p->mtCoder);
+    }
+
+    vt.Code = Lzma2Enc_MtCallback_Code;
+    vt.Write = Lzma2Enc_MtCallback_Write;
+
+    p->outStream = outStream;
+    p->outBuf = NULL;
+    p->outBufSize = 0;
+    if (!outStream)
+    {
+      p->outBuf = outBuf;
+      p->outBufSize = *outBufSize;
+      *outBufSize = 0;
+    }
+
+    p->mtCoder.allocBig = p->allocBig;
     p->mtCoder.progress = progress;
     p->mtCoder.inStream = inStream;
-    p->mtCoder.outStream = outStream;
-    p->mtCoder.alloc = p->alloc;
-    p->mtCoder.mtCallback = &mtCallback.funcTable;
+    p->mtCoder.inData = inData;
+    p->mtCoder.inDataSize = inDataSize;
+    p->mtCoder.mtCallback = &vt;
+    p->mtCoder.mtCallbackObject = p;
+
+    p->mtCoder.blockSize = (size_t)p->props.blockSize;
+    if (p->mtCoder.blockSize != p->props.blockSize)
+      return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */
 
-    p->mtCoder.blockSize = p->props.blockSize;
-    p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
-    if (p->mtCoder.destBlockSize < p->props.blockSize)
     {
-      p->mtCoder.destBlockSize = (size_t)0 - 1;
-      if (p->mtCoder.destBlockSize < p->props.blockSize)
-        return SZ_ERROR_FAIL;
+      size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16;
+      if (destBlockSize < p->mtCoder.blockSize)
+        return SZ_ERROR_PARAM;
+      if (p->outBufSize != destBlockSize)
+        Lzma2Enc_FreeOutBufs(p);
+      p->outBufSize = destBlockSize;
     }
-    p->mtCoder.numThreads = p->props.numBlockThreads;
+
+    p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max;
+    p->mtCoder.expectedDataSize = p->expectedDataSize;
     
-    return MtCoder_Code(&p->mtCoder);
+    {
+      SRes res = MtCoder_Code(&p->mtCoder);
+      if (!outStream)
+        *outBufSize = p->outBuf - outBuf;
+      return res;
+    }
   }
+
   #endif
 
-  return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
+
+  return Lzma2Enc_EncodeMt1(p,
+      &p->coders[0],
+      outStream, outBuf, outBufSize,
+      inStream, inData, inDataSize,
+      True, /* finished */
+      progress);
 }

C/Lzma2Enc.h

@@ -1,5 +1,5 @@
 /* Lzma2Enc.h -- LZMA2 Encoder
-2017-04-03 : Igor Pavlov : Public domain */
+2017-07-27 : Igor Pavlov : Public domain */
 
 #ifndef __LZMA2_ENC_H
 #define __LZMA2_ENC_H
@@ -8,11 +8,15 @@
 
 EXTERN_C_BEGIN
 
+#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0
+#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1)
+
 typedef struct
 {
   CLzmaEncProps lzmaProps;
-  size_t blockSize;
-  int numBlockThreads;
+  UInt64 blockSize;
+  int numBlockThreads_Reduced;
+  int numBlockThreads_Max;
   int numTotalThreads;
 } CLzma2EncProps;
 
@@ -22,13 +26,14 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p);
 /* ---------- CLzmaEnc2Handle Interface ---------- */
 
 /* Lzma2Enc_* functions can return the following exit codes:
-Returns:
+SRes:
   SZ_OK           - OK
   SZ_ERROR_MEM    - Memory allocation error
   SZ_ERROR_PARAM  - Incorrect paramater in props
-  SZ_ERROR_WRITE  - Write callback error
+  SZ_ERROR_WRITE  - ISeqOutStream write callback error
+  SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
   SZ_ERROR_PROGRESS - some break from progress callback
-  SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
+  SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
 */
 
 typedef void * CLzma2EncHandle;
@@ -36,26 +41,14 @@ typedef void * CLzma2EncHandle;
 CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig);
 void Lzma2Enc_Destroy(CLzma2EncHandle p);
 SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props);
+void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize);
 Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p);
-SRes Lzma2Enc_Encode(CLzma2EncHandle p,
-    ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
-
-/* ---------- One Call Interface ---------- */
-
-/* Lzma2Encode
-Return code:
-  SZ_OK               - OK
-  SZ_ERROR_MEM        - Memory allocation error
-  SZ_ERROR_PARAM      - Incorrect paramater
-  SZ_ERROR_OUTPUT_EOF - output buffer overflow
-  SZ_ERROR_THREAD     - errors in multithreading functions (only for Mt version)
-*/
-
-/*
-SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
-    const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark,
-    ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
-*/
+SRes Lzma2Enc_Encode2(CLzma2EncHandle p,
+    ISeqOutStream *outStream,
+    Byte *outBuf, size_t *outBufSize,
+    ISeqInStream *inStream,
+    const Byte *inData, size_t inDataSize,
+    ICompressProgress *progress);
 
 EXTERN_C_END
 

C/LzmaDec.c

@@ -1,8 +1,9 @@
 /* LzmaDec.c -- LZMA Decoder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-28 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
+/* #include "CpuArch.h" */
 #include "LzmaDec.h"
 
 #include <string.h>
@@ -24,9 +25,16 @@
 #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \
   { UPDATE_0(p); i = (i + i); A0; } else \
   { UPDATE_1(p); i = (i + i) + 1; A1; }
-#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;)
 
-#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); }
+#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); }
+
+#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \
+  { UPDATE_0(p + i); A0; } else \
+  { UPDATE_1(p + i); A1; }
+#define REV_BIT_VAR(  p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; )
+#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m;       , i += m * 2; )
+#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m        , ; )
+
 #define TREE_DECODE(probs, limit, i) \
   { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; }
 
@@ -46,12 +54,15 @@
   i -= 0x40; }
 #endif
 
-#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol)
+#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol)
 #define MATCHED_LITER_DEC \
-  matchByte <<= 1; \
-  bit = (matchByte & offs); \
-  probLit = prob + offs + bit + symbol; \
-  GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit)
+  matchByte += matchByte; \
+  bit = offs; \
+  offs &= matchByte; \
+  probLit = prob + (offs + bit + symbol); \
+  GET_BIT2(probLit, symbol, offs ^= bit; , ;)
+
+
 
 #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); }
 
@@ -66,25 +77,28 @@
   { i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; }
 
 
+#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \
+  { UPDATE_0_CHECK; i += m; m += m; } else \
+  { UPDATE_1_CHECK; m += m; i += m; }
+
+
 #define kNumPosBitsMax 4
 #define kNumPosStatesMax (1 << kNumPosBitsMax)
 
 #define kLenNumLowBits 3
 #define kLenNumLowSymbols (1 << kLenNumLowBits)
-#define kLenNumMidBits 3
-#define kLenNumMidSymbols (1 << kLenNumMidBits)
 #define kLenNumHighBits 8
 #define kLenNumHighSymbols (1 << kLenNumHighBits)
 
-#define LenChoice 0
-#define LenChoice2 (LenChoice + 1)
-#define LenLow (LenChoice2 + 1)
-#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits))
-#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits))
+#define LenLow 0
+#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits))
 #define kNumLenProbs (LenHigh + kLenNumHighSymbols)
 
+#define LenChoice LenLow
+#define LenChoice2 (LenLow + (1 << kLenNumLowBits))
 
 #define kNumStates 12
+#define kNumStates2 16
 #define kNumLitStates 7
 
 #define kStartPosModelIndex 4
@@ -98,54 +112,117 @@
 #define kAlignTableSize (1 << kNumAlignBits)
 
 #define kMatchMinLen 2
-#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols)
+#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols)
 
-#define IsMatch 0
-#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax))
+/* External ASM code needs same CLzmaProb array layout. So don't change it. */
+
+/* (probs_1664) is faster and better for code size at some platforms */
+/*
+#ifdef MY_CPU_X86_OR_AMD64
+*/
+#define kStartOffset 1664
+#define GET_PROBS p->probs_1664
+/*
+#define GET_PROBS p->probs + kStartOffset
+#else
+#define kStartOffset 0
+#define GET_PROBS p->probs
+#endif
+*/
+
+#define SpecPos (-kStartOffset)
+#define IsRep0Long (SpecPos + kNumFullDistances)
+#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax))
+#define LenCoder (RepLenCoder + kNumLenProbs)
+#define IsMatch (LenCoder + kNumLenProbs)
+#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax))
+#define IsRep (Align + kAlignTableSize)
 #define IsRepG0 (IsRep + kNumStates)
 #define IsRepG1 (IsRepG0 + kNumStates)
 #define IsRepG2 (IsRepG1 + kNumStates)
-#define IsRep0Long (IsRepG2 + kNumStates)
-#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax))
-#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits))
-#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex)
-#define LenCoder (Align + kAlignTableSize)
-#define RepLenCoder (LenCoder + kNumLenProbs)
-#define Literal (RepLenCoder + kNumLenProbs)
+#define PosSlot (IsRepG2 + kNumStates)
+#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits))
+#define NUM_BASE_PROBS (Literal + kStartOffset)
 
-#define LZMA_BASE_SIZE 1846
-#define LZMA_LIT_SIZE 0x300
-
-#if Literal != LZMA_BASE_SIZE
-StopCompilingDueBUG
+#if Align != 0 && kStartOffset != 0
+  #error Stop_Compiling_Bad_LZMA_kAlign
 #endif
 
-#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp)))
+#if NUM_BASE_PROBS != 1984
+  #error Stop_Compiling_Bad_LZMA_PROBS
+#endif
+
+
+#define LZMA_LIT_SIZE 0x300
+
+#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp)))
+
+
+#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4)
+#define COMBINED_PS_STATE (posState + state)
+#define GET_LEN_STATE (posState)
 
 #define LZMA_DIC_MIN (1 << 12)
 
-/* First LZMA-symbol is always decoded.
-And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization
+/*
+p->remainLen : shows status of LZMA decoder:
+    < kMatchSpecLenStart : normal remain
+    = kMatchSpecLenStart : finished
+    = kMatchSpecLenStart + 1 : need init range coder
+    = kMatchSpecLenStart + 2 : need init range coder and state
+*/
+
+/* ---------- LZMA_DECODE_REAL ---------- */
+/*
+LzmaDec_DecodeReal_3() can be implemented in external ASM file.
+3 - is the code compatibility version of that function for check at link time.
+*/
+
+#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3
+
+/*
+LZMA_DECODE_REAL()
+In:
+  RangeCoder is normalized
+  if (p->dicPos == limit)
+  {
+    LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases.
+    So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol
+    is not END_OF_PAYALOAD_MARKER, then function returns error code.
+  }
+
+Processing:
+  first LZMA symbol will be decoded in any case
+  All checks for limits are at the end of main loop,
+  It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit),
+  RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked.
+
 Out:
+  RangeCoder is normalized
   Result:
     SZ_OK - OK
     SZ_ERROR_DATA - Error
   p->remainLen:
     < kMatchSpecLenStart : normal remain
     = kMatchSpecLenStart : finished
-    = kMatchSpecLenStart + 1 : Flush marker (unused now)
-    = kMatchSpecLenStart + 2 : State Init Marker (unused now)
 */
 
-static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit)
-{
-  CLzmaProb *probs = p->probs;
 
-  unsigned state = p->state;
+#ifdef _LZMA_DEC_OPT
+
+int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit);
+
+#else
+
+static
+int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit)
+{
+  CLzmaProb *probs = GET_PROBS;
+  unsigned state = (unsigned)p->state;
   UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3];
   unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1;
-  unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1;
   unsigned lc = p->prop.lc;
+  unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc);
 
   Byte *dic = p->dic;
   SizeT dicBufSize = p->dicBufSize;
@@ -164,17 +241,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
     CLzmaProb *prob;
     UInt32 bound;
     unsigned ttt;
-    unsigned posState = processedPos & pbMask;
+    unsigned posState = CALC_POS_STATE(processedPos, pbMask);
 
-    prob = probs + IsMatch + (state << kNumPosBitsMax) + posState;
+    prob = probs + IsMatch + COMBINED_PS_STATE;
     IF_BIT_0(prob)
     {
       unsigned symbol;
       UPDATE_0(prob);
       prob = probs + Literal;
       if (processedPos != 0 || checkDicSize != 0)
-        prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) +
-            (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc))));
+        prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc);
       processedPos++;
 
       if (state < kNumLitStates)
@@ -240,13 +316,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
       else
       {
         UPDATE_1(prob);
+        /*
+        // that case was checked before with kBadRepCode
         if (checkDicSize == 0 && processedPos == 0)
           return SZ_ERROR_DATA;
+        */
         prob = probs + IsRepG0 + state;
         IF_BIT_0(prob)
         {
           UPDATE_0(prob);
-          prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState;
+          prob = probs + IsRep0Long + COMBINED_PS_STATE;
           IF_BIT_0(prob)
           {
             UPDATE_0(prob);
@@ -299,7 +378,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
         IF_BIT_0(probLen)
         {
           UPDATE_0(probLen);
-          probLen = prob + LenLow + (posState << kLenNumLowBits);
+          probLen = prob + LenLow + GET_LEN_STATE;
           offset = 0;
           lim = (1 << kLenNumLowBits);
         }
@@ -310,15 +389,15 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
           IF_BIT_0(probLen)
           {
             UPDATE_0(probLen);
-            probLen = prob + LenMid + (posState << kLenNumMidBits);
+            probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits);
             offset = kLenNumLowSymbols;
-            lim = (1 << kLenNumMidBits);
+            lim = (1 << kLenNumLowBits);
           }
           else
           {
             UPDATE_1(probLen);
             probLen = prob + LenHigh;
-            offset = kLenNumLowSymbols + kLenNumMidSymbols;
+            offset = kLenNumLowSymbols * 2;
             lim = (1 << kLenNumHighBits);
           }
         }
@@ -331,7 +410,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
         IF_BIT_0(probLen)
         {
           UPDATE_0(probLen);
-          probLen = prob + LenLow + (posState << kLenNumLowBits);
+          probLen = prob + LenLow + GET_LEN_STATE;
           len = 1;
           TREE_GET_BIT(probLen, len);
           TREE_GET_BIT(probLen, len);
@@ -345,7 +424,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
           IF_BIT_0(probLen)
           {
             UPDATE_0(probLen);
-            probLen = prob + LenMid + (posState << kLenNumMidBits);
+            probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits);
             len = 1;
             TREE_GET_BIT(probLen, len);
             TREE_GET_BIT(probLen, len);
@@ -356,7 +435,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
             UPDATE_1(probLen);
             probLen = prob + LenHigh;
             TREE_DECODE(probLen, (1 << kLenNumHighBits), len);
-            len += kLenNumLowSymbols + kLenNumMidSymbols;
+            len += kLenNumLowSymbols * 2;
           }
         }
       }
@@ -376,16 +455,16 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
           if (posSlot < kEndPosModelIndex)
           {
             distance <<= numDirectBits;
-            prob = probs + SpecPos + distance - posSlot - 1;
+            prob = probs + SpecPos;
             {
-              UInt32 mask = 1;
-              unsigned i = 1;
+              UInt32 m = 1;
+              distance++;
               do
               {
-                GET_BIT2(prob + i, i, ; , distance |= mask);
-                mask <<= 1;
+                REV_BIT_VAR(prob, distance, m);
               }
-              while (--numDirectBits != 0);
+              while (--numDirectBits);
+              distance -= m;
             }
           }
           else
@@ -412,19 +491,20 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
               }
               */
             }
-            while (--numDirectBits != 0);
+            while (--numDirectBits);
             prob = probs + Align;
             distance <<= kNumAlignBits;
             {
               unsigned i = 1;
-              GET_BIT2(prob + i, i, ; , distance |= 1);
-              GET_BIT2(prob + i, i, ; , distance |= 2);
-              GET_BIT2(prob + i, i, ; , distance |= 4);
-              GET_BIT2(prob + i, i, ; , distance |= 8);
+              REV_BIT_CONST(prob, i, 1);
+              REV_BIT_CONST(prob, i, 2);
+              REV_BIT_CONST(prob, i, 4);
+              REV_BIT_LAST (prob, i, 8);
+              distance |= i;
             }
             if (distance == (UInt32)0xFFFFFFFF)
             {
-              len += kMatchSpecLenStart;
+              len = kMatchSpecLenStart;
               state -= kNumStates;
               break;
             }
@@ -435,20 +515,12 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
         rep2 = rep1;
         rep1 = rep0;
         rep0 = distance + 1;
-        if (checkDicSize == 0)
-        {
-          if (distance >= processedPos)
-          {
-            p->dicPos = dicPos;
-            return SZ_ERROR_DATA;
-          }
-        }
-        else if (distance >= checkDicSize)
+        state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3;
+        if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize))
         {
           p->dicPos = dicPos;
           return SZ_ERROR_DATA;
         }
-        state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3;
       }
 
       len += kMatchMinLen;
@@ -511,6 +583,7 @@ static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte
 
   return SZ_OK;
 }
+#endif
 
 static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit)
 {
@@ -519,7 +592,7 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit)
     Byte *dic = p->dic;
     SizeT dicPos = p->dicPos;
     SizeT dicBufSize = p->dicBufSize;
-    unsigned len = p->remainLen;
+    unsigned len = (unsigned)p->remainLen;
     SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */
     SizeT rem = limit - dicPos;
     if (rem < len)
@@ -540,6 +613,14 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit)
   }
 }
 
+
+#define kRange0 0xFFFFFFFF
+#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))
+#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)))
+#if kBadRepCode != (0xC0000000 - 0x400)
+  #error Stop_Compiling_Bad_LZMA_Check
+#endif
+
 static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit)
 {
   do
@@ -550,9 +631,13 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte
       UInt32 rem = p->prop.dicSize - p->processedPos;
       if (limit - p->dicPos > rem)
         limit2 = p->dicPos + rem;
+
+      if (p->processedPos == 0)
+        if (p->code >= kBadRepCode)
+          return SZ_ERROR_DATA;
     }
-    
-    RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit));
+
+    RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit));
     
     if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize)
       p->checkDicSize = p->prop.dicSize;
@@ -561,9 +646,6 @@ static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte
   }
   while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart);
 
-  if (p->remainLen > kMatchSpecLenStart)
-    p->remainLen = kMatchSpecLenStart;
-
   return 0;
 }
 
@@ -580,17 +662,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
   UInt32 range = p->range;
   UInt32 code = p->code;
   const Byte *bufLimit = buf + inSize;
-  const CLzmaProb *probs = p->probs;
-  unsigned state = p->state;
+  const CLzmaProb *probs = GET_PROBS;
+  unsigned state = (unsigned)p->state;
   ELzmaDummy res;
 
   {
     const CLzmaProb *prob;
     UInt32 bound;
     unsigned ttt;
-    unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1);
+    unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1);
 
-    prob = probs + IsMatch + (state << kNumPosBitsMax) + posState;
+    prob = probs + IsMatch + COMBINED_PS_STATE;
     IF_BIT_0_CHECK(prob)
     {
       UPDATE_0_CHECK
@@ -618,10 +700,11 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
         {
           unsigned bit;
           const CLzmaProb *probLit;
-          matchByte <<= 1;
-          bit = (matchByte & offs);
-          probLit = prob + offs + bit + symbol;
-          GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit)
+          matchByte += matchByte;
+          bit = offs;
+          offs &= matchByte;
+          probLit = prob + (offs + bit + symbol);
+          GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; )
         }
         while (symbol < 0x100);
       }
@@ -648,7 +731,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
         IF_BIT_0_CHECK(prob)
         {
           UPDATE_0_CHECK;
-          prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState;
+          prob = probs + IsRep0Long + COMBINED_PS_STATE;
           IF_BIT_0_CHECK(prob)
           {
             UPDATE_0_CHECK;
@@ -691,7 +774,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
         IF_BIT_0_CHECK(probLen)
         {
           UPDATE_0_CHECK;
-          probLen = prob + LenLow + (posState << kLenNumLowBits);
+          probLen = prob + LenLow + GET_LEN_STATE;
           offset = 0;
           limit = 1 << kLenNumLowBits;
         }
@@ -702,15 +785,15 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
           IF_BIT_0_CHECK(probLen)
           {
             UPDATE_0_CHECK;
-            probLen = prob + LenMid + (posState << kLenNumMidBits);
+            probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits);
             offset = kLenNumLowSymbols;
-            limit = 1 << kLenNumMidBits;
+            limit = 1 << kLenNumLowBits;
           }
           else
           {
             UPDATE_1_CHECK;
             probLen = prob + LenHigh;
-            offset = kLenNumLowSymbols + kLenNumMidSymbols;
+            offset = kLenNumLowSymbols * 2;
             limit = 1 << kLenNumHighBits;
           }
         }
@@ -722,7 +805,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
       {
         unsigned posSlot;
         prob = probs + PosSlot +
-            ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) <<
+            ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) <<
             kNumPosSlotBits);
         TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot);
         if (posSlot >= kStartPosModelIndex)
@@ -733,7 +816,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
 
           if (posSlot < kEndPosModelIndex)
           {
-            prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1;
+            prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits);
           }
           else
           {
@@ -745,17 +828,18 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
               code -= range & (((code - range) >> 31) - 1);
               /* if (code >= range) code -= range; */
             }
-            while (--numDirectBits != 0);
+            while (--numDirectBits);
             prob = probs + Align;
             numDirectBits = kNumAlignBits;
           }
           {
             unsigned i = 1;
+            unsigned m = 1;
             do
             {
-              GET_BIT_CHECK(prob + i, i);
+              REV_BIT_CHECK(prob, i, m);
             }
-            while (--numDirectBits != 0);
+            while (--numDirectBits);
           }
         }
       }
@@ -768,18 +852,17 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
 
 void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState)
 {
-  p->needFlush = 1;
-  p->remainLen = 0;
+  p->remainLen = kMatchSpecLenStart + 1;
   p->tempBufSize = 0;
 
   if (initDic)
   {
     p->processedPos = 0;
     p->checkDicSize = 0;
-    p->needInitState = 1;
+    p->remainLen = kMatchSpecLenStart + 2;
   }
   if (initState)
-    p->needInitState = 1;
+    p->remainLen = kMatchSpecLenStart + 2;
 }
 
 void LzmaDec_Init(CLzmaDec *p)
@@ -788,53 +871,54 @@ void LzmaDec_Init(CLzmaDec *p)
   LzmaDec_InitDicAndState(p, True, True);
 }
 
-static void LzmaDec_InitStateReal(CLzmaDec *p)
-{
-  SizeT numProbs = LzmaProps_GetNumProbs(&p->prop);
-  SizeT i;
-  CLzmaProb *probs = p->probs;
-  for (i = 0; i < numProbs; i++)
-    probs[i] = kBitModelTotal >> 1;
-  p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1;
-  p->state = 0;
-  p->needInitState = 0;
-}
 
 SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen,
     ELzmaFinishMode finishMode, ELzmaStatus *status)
 {
   SizeT inSize = *srcLen;
   (*srcLen) = 0;
-  LzmaDec_WriteRem(p, dicLimit);
   
   *status = LZMA_STATUS_NOT_SPECIFIED;
 
+  if (p->remainLen > kMatchSpecLenStart)
+  {
+    for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--)
+      p->tempBuf[p->tempBufSize++] = *src++;
+    if (p->tempBufSize != 0 && p->tempBuf[0] != 0)
+      return SZ_ERROR_DATA;
+    if (p->tempBufSize < RC_INIT_SIZE)
+    {
+      *status = LZMA_STATUS_NEEDS_MORE_INPUT;
+      return SZ_OK;
+    }
+    p->code =
+        ((UInt32)p->tempBuf[1] << 24)
+      | ((UInt32)p->tempBuf[2] << 16)
+      | ((UInt32)p->tempBuf[3] << 8)
+      | ((UInt32)p->tempBuf[4]);
+    p->range = 0xFFFFFFFF;
+    p->tempBufSize = 0;
+
+    if (p->remainLen > kMatchSpecLenStart + 1)
+    {
+      SizeT numProbs = LzmaProps_GetNumProbs(&p->prop);
+      SizeT i;
+      CLzmaProb *probs = p->probs;
+      for (i = 0; i < numProbs; i++)
+        probs[i] = kBitModelTotal >> 1;
+      p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1;
+      p->state = 0;
+    }
+
+    p->remainLen = 0;
+  }
+
+  LzmaDec_WriteRem(p, dicLimit);
+
   while (p->remainLen != kMatchSpecLenStart)
   {
-      int checkEndMarkNow;
+      int checkEndMarkNow = 0;
 
-      if (p->needFlush)
-      {
-        for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--)
-          p->tempBuf[p->tempBufSize++] = *src++;
-        if (p->tempBufSize < RC_INIT_SIZE)
-        {
-          *status = LZMA_STATUS_NEEDS_MORE_INPUT;
-          return SZ_OK;
-        }
-        if (p->tempBuf[0] != 0)
-          return SZ_ERROR_DATA;
-        p->code =
-              ((UInt32)p->tempBuf[1] << 24)
-            | ((UInt32)p->tempBuf[2] << 16)
-            | ((UInt32)p->tempBuf[3] << 8)
-            | ((UInt32)p->tempBuf[4]);
-        p->range = 0xFFFFFFFF;
-        p->needFlush = 0;
-        p->tempBufSize = 0;
-      }
-
-      checkEndMarkNow = 0;
       if (p->dicPos >= dicLimit)
       {
         if (p->remainLen == 0 && p->code == 0)
@@ -855,9 +939,6 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr
         checkEndMarkNow = 1;
       }
 
-      if (p->needInitState)
-        LzmaDec_InitStateReal(p);
-  
       if (p->tempBufSize == 0)
       {
         SizeT processed;
@@ -930,11 +1011,14 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr
         p->tempBufSize = 0;
       }
   }
-  if (p->code == 0)
-    *status = LZMA_STATUS_FINISHED_WITH_MARK;
-  return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA;
+  
+  if (p->code != 0)
+    return SZ_ERROR_DATA;
+  *status = LZMA_STATUS_FINISHED_WITH_MARK;
+  return SZ_OK;
 }
 
+
 SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status)
 {
   SizeT outSize = *destLen;
@@ -1011,10 +1095,10 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size)
   if (d >= (9 * 5 * 5))
     return SZ_ERROR_UNSUPPORTED;
 
-  p->lc = d % 9;
+  p->lc = (Byte)(d % 9);
   d /= 9;
-  p->pb = d / 5;
-  p->lp = d % 5;
+  p->pb = (Byte)(d / 5);
+  p->lp = (Byte)(d % 5);
 
   return SZ_OK;
 }
@@ -1026,9 +1110,10 @@ static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAl
   {
     LzmaDec_FreeProbs(p, alloc);
     p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb));
-    p->numProbs = numProbs;
     if (!p->probs)
       return SZ_ERROR_MEM;
+    p->probs_1664 = p->probs + 1664;
+    p->numProbs = numProbs;
   }
   return SZ_OK;
 }

C/LzmaDec.h

@@ -1,5 +1,5 @@
 /* LzmaDec.h -- LZMA Decoder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-06 : Igor Pavlov : Public domain */
 
 #ifndef __LZMA_DEC_H
 #define __LZMA_DEC_H
@@ -25,7 +25,10 @@ EXTERN_C_BEGIN
 
 typedef struct _CLzmaProps
 {
-  unsigned lc, lp, pb;
+  Byte lc;
+  Byte lp;
+  Byte pb;
+  Byte _pad_;
   UInt32 dicSize;
 } CLzmaProps;
 
@@ -47,26 +50,28 @@ SRes LzmaProps_Decode(CLzmaProps *p, const Byte *data, unsigned size);
 
 typedef struct
 {
+  /* Don't change this structure. ASM code can use it. */
   CLzmaProps prop;
   CLzmaProb *probs;
+  CLzmaProb *probs_1664;
   Byte *dic;
-  const Byte *buf;
-  UInt32 range, code;
-  SizeT dicPos;
   SizeT dicBufSize;
+  SizeT dicPos;
+  const Byte *buf;
+  UInt32 range;
+  UInt32 code;
   UInt32 processedPos;
   UInt32 checkDicSize;
-  unsigned state;
   UInt32 reps[4];
-  unsigned remainLen;
-  int needFlush;
-  int needInitState;
+  UInt32 state;
+  UInt32 remainLen;
+
   UInt32 numProbs;
   unsigned tempBufSize;
   Byte tempBuf[LZMA_REQUIRED_INPUT_MAX];
 } CLzmaDec;
 
-#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; }
+#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; }
 
 void LzmaDec_Init(CLzmaDec *p);
 
@@ -132,8 +137,8 @@ LzmaDec_Allocate* can return:
 SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc);
 void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc);
 
-SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAllocPtr alloc);
-void LzmaDec_Free(CLzmaDec *state, ISzAllocPtr alloc);
+SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc);
+void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc);
 
 /* ---------- Dictionary Interface ---------- */
 

C/LzmaEnc.c

@@ -1,5 +1,5 @@
 /* LzmaEnc.c -- LZMA Encoder
-2017-04-03 : Igor Pavlov : Public domain */
+2017-06-22 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -23,8 +23,8 @@
 static unsigned g_STAT_OFFSET = 0;
 #endif
 
-#define kMaxHistorySize ((UInt32)3 << 29)
-/* #define kMaxHistorySize ((UInt32)7 << 29) */
+#define kLzmaMaxHistorySize ((UInt32)3 << 29)
+/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */
 
 #define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1)
 
@@ -62,14 +62,15 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p)
   if (level < 0) level = 5;
   p->level = level;
   
-  if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26)));
+  if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26)));
   if (p->dictSize > p->reduceSize)
   {
     unsigned i;
+    UInt32 reduceSize = (UInt32)p->reduceSize;
     for (i = 11; i <= 30; i++)
     {
-      if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; }
-      if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; }
+      if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; }
+      if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; }
     }
   }
 
@@ -445,7 +446,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
       || props.lp > LZMA_LP_MAX
       || props.pb > LZMA_PB_MAX
       || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress)
-      || props.dictSize > kMaxHistorySize)
+      || props.dictSize > kLzmaMaxHistorySize)
     return SZ_ERROR_PARAM;
 
   p->dictSize = props.dictSize;
@@ -492,6 +493,15 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
   return SZ_OK;
 }
 
+
+void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize)
+{
+  CLzmaEnc *p = (CLzmaEnc *)pp;
+  p->matchFinderBase.expectedDataSize = expectedDataSiize;
+}
+
+
+
 static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4,  5,  6,   4, 5};
 static const int kMatchNextStates[kNumStates]   = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10};
 static const int kRepNextStates[kNumStates]     = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11};
@@ -1996,6 +2006,8 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc,
   {
     RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig));
     p->matchFinderObj = &p->matchFinderMt;
+    p->matchFinderBase.bigHash = (Byte)(
+        (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0);
     MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder);
   }
   else
@@ -2135,6 +2147,7 @@ SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
   LzmaEnc_SetInputBuf(p, src, srcLen);
   p->needInit = 1;
 
+  LzmaEnc_SetDataSize(pp, srcLen);
   return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig);
 }
 
@@ -2152,15 +2165,15 @@ void LzmaEnc_Finish(CLzmaEncHandle pp)
 
 typedef struct
 {
-  ISeqOutStream funcTable;
+  ISeqOutStream vt;
   Byte *data;
   SizeT rem;
   Bool overflow;
-} CSeqOutStreamBuf;
+} CLzmaEnc_SeqOutStreamBuf;
 
-static size_t MyWrite(const ISeqOutStream *pp, const void *data, size_t size)
+static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size)
 {
-  CSeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CSeqOutStreamBuf, funcTable);
+  CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt);
   if (p->rem < size)
   {
     size = p->rem;
@@ -2193,9 +2206,9 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
   CLzmaEnc *p = (CLzmaEnc *)pp;
   UInt64 nowPos64;
   SRes res;
-  CSeqOutStreamBuf outStream;
+  CLzmaEnc_SeqOutStreamBuf outStream;
 
-  outStream.funcTable.Write = MyWrite;
+  outStream.vt.Write = SeqOutStreamBuf_Write;
   outStream.data = dest;
   outStream.rem = *destLen;
   outStream.overflow = False;
@@ -2209,7 +2222,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
   LzmaEnc_InitPrices(p);
   nowPos64 = p->nowPos64;
   RangeEnc_Init(&p->rc);
-  p->rc.outStream = &outStream.funcTable;
+  p->rc.outStream = &outStream.vt;
 
   res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize);
   
@@ -2308,15 +2321,15 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte
   SRes res;
   CLzmaEnc *p = (CLzmaEnc *)pp;
 
-  CSeqOutStreamBuf outStream;
+  CLzmaEnc_SeqOutStreamBuf outStream;
 
-  outStream.funcTable.Write = MyWrite;
+  outStream.vt.Write = SeqOutStreamBuf_Write;
   outStream.data = dest;
   outStream.rem = *destLen;
   outStream.overflow = False;
 
   p->writeEndMark = writeEndMark;
-  p->rc.outStream = &outStream.funcTable;
+  p->rc.outStream = &outStream.vt;
 
   res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig);
   

C/LzmaEnc.h

@@ -1,5 +1,5 @@
 /*  LzmaEnc.h -- LZMA Encoder
-2017-04-03 : Igor Pavlov : Public domain */
+2017-07-27 : Igor Pavlov : Public domain */
 
 #ifndef __LZMA_ENC_H
 #define __LZMA_ENC_H
@@ -12,12 +12,10 @@ EXTERN_C_BEGIN
 
 typedef struct _CLzmaEncProps
 {
-  int level;       /*  0 <= level <= 9 */
+  int level;       /* 0 <= level <= 9 */
   UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version
-                      (1 << 12) <= dictSize <= (1 << 30) for 64-bit version
-                       default = (1 << 24) */
-  UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF.
-                        Encoder uses this value to reduce dictionary size */
+                      (1 << 12) <= dictSize <= (3 << 29) for 64-bit version
+                      default = (1 << 24) */
   int lc;          /* 0 <= lc <= 8, default = 3 */
   int lp;          /* 0 <= lp <= 4, default = 0 */
   int pb;          /* 0 <= pb <= 4, default = 2 */
@@ -25,9 +23,12 @@ typedef struct _CLzmaEncProps
   int fb;          /* 5 <= fb <= 273, default = 32 */
   int btMode;      /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */
   int numHashBytes; /* 2, 3 or 4, default = 4 */
-  UInt32 mc;        /* 1 <= mc <= (1 << 30), default = 32 */
+  UInt32 mc;       /* 1 <= mc <= (1 << 30), default = 32 */
   unsigned writeEndMark;  /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */
   int numThreads;  /* 1 or 2, default = 2 */
+
+  UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1.
+                        Encoder uses this value to reduce dictionary size */
 } CLzmaEncProps;
 
 void LzmaEncProps_Init(CLzmaEncProps *p);
@@ -37,38 +38,34 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2);
 
 /* ---------- CLzmaEncHandle Interface ---------- */
 
-/* LzmaEnc_* functions can return the following exit codes:
-Returns:
+/* LzmaEnc* functions can return the following exit codes:
+SRes:
   SZ_OK           - OK
   SZ_ERROR_MEM    - Memory allocation error
   SZ_ERROR_PARAM  - Incorrect paramater in props
-  SZ_ERROR_WRITE  - Write callback error.
+  SZ_ERROR_WRITE  - ISeqOutStream write callback error
+  SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
   SZ_ERROR_PROGRESS - some break from progress callback
-  SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
+  SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
 */
 
 typedef void * CLzmaEncHandle;
 
 CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc);
 void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig);
+
 SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props);
+void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize);
 SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size);
 unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p);
+
 SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream,
     ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
 SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
     int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
 
-/* ---------- One Call Interface ---------- */
 
-/* LzmaEncode
-Return code:
-  SZ_OK               - OK
-  SZ_ERROR_MEM        - Memory allocation error
-  SZ_ERROR_PARAM      - Incorrect paramater
-  SZ_ERROR_OUTPUT_EOF - output buffer overflow
-  SZ_ERROR_THREAD     - errors in multithreading functions (only for Mt version)
-*/
+/* ---------- One Call Interface ---------- */
 
 SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
     const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark,

C/MtCoder.c

@@ -1,168 +1,89 @@
 /* MtCoder.c -- Multi-thread Coder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-21 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
 #include "MtCoder.h"
 
-void LoopThread_Construct(CLoopThread *p)
-{
-  Thread_Construct(&p->thread);
-  Event_Construct(&p->startEvent);
-  Event_Construct(&p->finishedEvent);
-}
+#ifndef _7ZIP_ST
 
-void LoopThread_Close(CLoopThread *p)
+SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize)
 {
-  Thread_Close(&p->thread);
-  Event_Close(&p->startEvent);
-  Event_Close(&p->finishedEvent);
-}
-
-static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp)
-{
-  CLoopThread *p = (CLoopThread *)pp;
-  for (;;)
+  CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt);
+  UInt64 inSize2 = 0;
+  UInt64 outSize2 = 0;
+  if (inSize != (UInt64)(Int64)-1)
   {
-    if (Event_Wait(&p->startEvent) != 0)
-      return SZ_ERROR_THREAD;
-    if (p->stop)
-      return 0;
-    p->res = p->func(p->param);
-    if (Event_Set(&p->finishedEvent) != 0)
-      return SZ_ERROR_THREAD;
+    inSize2 = inSize - thunk->inSize;
+    thunk->inSize = inSize;
   }
+  if (outSize != (UInt64)(Int64)-1)
+  {
+    outSize2 = outSize - thunk->outSize;
+    thunk->outSize = outSize;
+  }
+  return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2);
 }
 
-WRes LoopThread_Create(CLoopThread *p)
+
+void MtProgressThunk_CreateVTable(CMtProgressThunk *p)
 {
-  p->stop = 0;
-  RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent));
-  RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent));
-  return Thread_Create(&p->thread, LoopThreadFunc, p);
+  p->vt.Progress = MtProgressThunk_Progress;
 }
 
-WRes LoopThread_StopAndWait(CLoopThread *p)
-{
-  p->stop = 1;
-  if (Event_Set(&p->startEvent) != 0)
-    return SZ_ERROR_THREAD;
-  return Thread_Wait(&p->thread);
-}
 
-WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); }
-WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); }
-
-static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
-{
-  return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
-}
-
-static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress)
-{
-  unsigned i;
-  for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
-    p->inSizes[i] = p->outSizes[i] = 0;
-  p->totalInSize = p->totalOutSize = 0;
-  p->progress = progress;
-  p->res = SZ_OK;
-}
-
-static void MtProgress_Reinit(CMtProgress *p, unsigned index)
-{
-  p->inSizes[index] = 0;
-  p->outSizes[index] = 0;
-}
-
-#define UPDATE_PROGRESS(size, prev, total) \
-  if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; }
-
-SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize)
-{
-  SRes res;
-  CriticalSection_Enter(&p->cs);
-  UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize)
-  UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize)
-  if (p->res == SZ_OK)
-    p->res = Progress(p->progress, p->totalInSize, p->totalOutSize);
-  res = p->res;
-  CriticalSection_Leave(&p->cs);
-  return res;
-}
-
-static void MtProgress_SetError(CMtProgress *p, SRes res)
-{
-  CriticalSection_Enter(&p->cs);
-  if (p->res == SZ_OK)
-    p->res = res;
-  CriticalSection_Leave(&p->cs);
-}
-
-static void MtCoder_SetError(CMtCoder* p, SRes res)
-{
-  CriticalSection_Enter(&p->cs);
-  if (p->res == SZ_OK)
-    p->res = res;
-  CriticalSection_Leave(&p->cs);
-}
-
-/* ---------- MtThread ---------- */
-
-void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder)
-{
-  p->mtCoder = mtCoder;
-  p->outBuf = 0;
-  p->inBuf = 0;
-  Event_Construct(&p->canRead);
-  Event_Construct(&p->canWrite);
-  LoopThread_Construct(&p->thread);
-}
 
 #define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; }
 
-static void CMtThread_CloseEvents(CMtThread *p)
+
+static WRes ArEvent_OptCreate_And_Reset(CEvent *p)
 {
-  Event_Close(&p->canRead);
-  Event_Close(&p->canWrite);
+  if (Event_IsCreated(p))
+    return Event_Reset(p);
+  return AutoResetEvent_CreateNotSignaled(p);
 }
 
-static void CMtThread_Destruct(CMtThread *p)
-{
-  CMtThread_CloseEvents(p);
 
-  if (Thread_WasCreated(&p->thread.thread))
+static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp);
+
+
+static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t)
+{
+  WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent);
+  if (wres == 0)
   {
-    LoopThread_StopAndWait(&p->thread);
-    LoopThread_Close(&p->thread);
+    t->stop = False;
+    if (!Thread_WasCreated(&t->thread))
+      wres = Thread_Create(&t->thread, ThreadFunc, t);
+    if (wres == 0)
+      wres = Event_Set(&t->startEvent);
+  }
+  if (wres == 0)
+    return SZ_OK;
+  return MY_SRes_HRESULT_FROM_WRes(wres);
+}
+
+
+static void MtCoderThread_Destruct(CMtCoderThread *t)
+{
+  if (Thread_WasCreated(&t->thread))
+  {
+    t->stop = 1;
+    Event_Set(&t->startEvent);
+    Thread_Wait(&t->thread);
+    Thread_Close(&t->thread);
   }
 
-  if (p->mtCoder->alloc)
-    ISzAlloc_Free(p->mtCoder->alloc, p->outBuf);
-  p->outBuf = 0;
+  Event_Close(&t->startEvent);
 
-  if (p->mtCoder->alloc)
-    ISzAlloc_Free(p->mtCoder->alloc, p->inBuf);
-  p->inBuf = 0;
+  if (t->inBuf)
+  {
+    ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf);
+    t->inBuf = NULL;
+  }
 }
 
-#define MY_BUF_ALLOC(buf, size, newSize) \
-  if (buf == 0 || size != newSize) \
-  { ISzAlloc_Free(p->mtCoder->alloc, buf); \
-    size = newSize; buf = (Byte *)ISzAlloc_Alloc(p->mtCoder->alloc, size); \
-    if (buf == 0) return SZ_ERROR_MEM; }
 
-static SRes CMtThread_Prepare(CMtThread *p)
-{
-  MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize)
-  MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize)
-
-  p->stopReading = False;
-  p->stopWriting = False;
-  RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead));
-  RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite));
-
-  return SZ_OK;
-}
 
 static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
 {
@@ -170,158 +91,511 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
   *processedSize = 0;
   while (size != 0)
   {
-    size_t curSize = size;
-    SRes res = ISeqInStream_Read(stream, data, &curSize);
-    *processedSize += curSize;
-    data += curSize;
-    size -= curSize;
+    size_t cur = size;
+    SRes res = ISeqInStream_Read(stream, data, &cur);
+    *processedSize += cur;
+    data += cur;
+    size -= cur;
     RINOK(res);
-    if (curSize == 0)
+    if (cur == 0)
       return SZ_OK;
   }
   return SZ_OK;
 }
 
-#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads  - 1 ? 0 : p->index + 1]
 
-static SRes MtThread_Process(CMtThread *p, Bool *stop)
+/*
+  ThreadFunc2() returns:
+  SZ_OK           - in all normal cases (even for stream error or memory allocation error)
+  SZ_ERROR_THREAD - in case of failure in system synch function
+*/
+
+static SRes ThreadFunc2(CMtCoderThread *t)
 {
-  CMtThread *next;
-  *stop = True;
-  if (Event_Wait(&p->canRead) != 0)
-    return SZ_ERROR_THREAD;
-  
-  next = GET_NEXT_THREAD(p);
-  
-  if (p->stopReading)
-  {
-    next->stopReading = True;
-    return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD;
-  }
+  CMtCoder *mtc = t->mtCoder;
 
-  {
-    size_t size = p->mtCoder->blockSize;
-    size_t destSize = p->outBufSize;
-
-    RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size));
-    next->stopReading = *stop = (size != p->mtCoder->blockSize);
-    if (Event_Set(&next->canRead) != 0)
-      return SZ_ERROR_THREAD;
-
-    RINOK(IMtCoderCallback_Code(p->mtCoder->mtCallback, p->index,
-        p->outBuf, &destSize, p->inBuf, size, *stop));
-
-    MtProgress_Reinit(&p->mtCoder->mtProgress, p->index);
-
-    if (Event_Wait(&p->canWrite) != 0)
-      return SZ_ERROR_THREAD;
-    if (p->stopWriting)
-      return SZ_ERROR_FAIL;
-    if (ISeqOutStream_Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize)
-      return SZ_ERROR_WRITE;
-    return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD;
-  }
-}
-
-static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
-{
-  CMtThread *p = (CMtThread *)pp;
   for (;;)
   {
-    Bool stop;
-    CMtThread *next = GET_NEXT_THREAD(p);
-    SRes res = MtThread_Process(p, &stop);
-    if (res != SZ_OK)
+    unsigned bi;
+    SRes res;
+    SRes res2;
+    Bool finished;
+    unsigned bufIndex;
+    size_t size;
+    const Byte *inData;
+    UInt64 readProcessed = 0;
+    
+    RINOK_THREAD(Event_Wait(&mtc->readEvent))
+
+    /* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */
+
+    if (mtc->stopReading)
     {
-      MtCoder_SetError(p->mtCoder, res);
-      MtProgress_SetError(&p->mtCoder->mtProgress, res);
-      next->stopReading = True;
-      next->stopWriting = True;
-      Event_Set(&next->canRead);
-      Event_Set(&next->canWrite);
-      return res;
+      return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD;
     }
-    if (stop)
+
+    res = MtProgress_GetError(&mtc->mtProgress);
+    
+    size = 0;
+    inData = NULL;
+    finished = True;
+
+    if (res == SZ_OK)
+    {
+      size = mtc->blockSize;
+      if (mtc->inStream)
+      {
+        if (!t->inBuf)
+        {
+          t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize);
+          if (!t->inBuf)
+            res = SZ_ERROR_MEM;
+        }
+        if (res == SZ_OK)
+        {
+          res = FullRead(mtc->inStream, t->inBuf, &size);
+          readProcessed = mtc->readProcessed + size;
+          mtc->readProcessed = readProcessed;
+        }
+        if (res != SZ_OK)
+        {
+          mtc->readRes = res;
+          /* after reading error - we can stop encoding of previous blocks */
+          MtProgress_SetError(&mtc->mtProgress, res);
+        }
+        else
+          finished = (size != mtc->blockSize);
+      }
+      else
+      {
+        size_t rem;
+        readProcessed = mtc->readProcessed;
+        rem = mtc->inDataSize - (size_t)readProcessed;
+        if (size > rem)
+          size = rem;
+        inData = mtc->inData + (size_t)readProcessed;
+        readProcessed += size;
+        mtc->readProcessed = readProcessed;
+        finished = (mtc->inDataSize == (size_t)readProcessed);
+      }
+    }
+
+    /* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */
+
+    res2 = SZ_OK;
+
+    if (Semaphore_Wait(&mtc->blocksSemaphore) != 0)
+    {
+      res2 = SZ_ERROR_THREAD;
+      if (res == SZ_OK)
+      {
+        res = res2;
+        // MtProgress_SetError(&mtc->mtProgress, res);
+      }
+    }
+
+    bi = mtc->blockIndex;
+
+    if (++mtc->blockIndex >= mtc->numBlocksMax)
+      mtc->blockIndex = 0;
+
+    bufIndex = (unsigned)(int)-1;
+
+    if (res == SZ_OK)
+      res = MtProgress_GetError(&mtc->mtProgress);
+
+    if (res != SZ_OK)
+      finished = True;
+
+    if (!finished)
+    {
+      if (mtc->numStartedThreads < mtc->numStartedThreadsLimit
+          && mtc->expectedDataSize != readProcessed)
+      {
+        res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]);
+        if (res == SZ_OK)
+          mtc->numStartedThreads++;
+        else
+        {
+          MtProgress_SetError(&mtc->mtProgress, res);
+          finished = True;
+        }
+      }
+    }
+
+    if (finished)
+      mtc->stopReading = True;
+
+    RINOK_THREAD(Event_Set(&mtc->readEvent))
+
+    if (res2 != SZ_OK)
+      return res2;
+
+    if (res == SZ_OK)
+    {
+      CriticalSection_Enter(&mtc->cs);
+      bufIndex = mtc->freeBlockHead;
+      mtc->freeBlockHead = mtc->freeBlockList[bufIndex];
+      CriticalSection_Leave(&mtc->cs);
+      
+      res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex,
+          mtc->inStream ? t->inBuf : inData, size, finished);
+      
+      // MtProgress_Reinit(&mtc->mtProgress, t->index);
+
+      if (res != SZ_OK)
+        MtProgress_SetError(&mtc->mtProgress, res);
+    }
+
+    {
+      CMtCoderBlock *block = &mtc->blocks[bi];
+      block->res = res;
+      block->bufIndex = bufIndex;
+      block->finished = finished;
+    }
+    
+    #ifdef MTCODER__USE_WRITE_THREAD
+      RINOK_THREAD(Event_Set(&mtc->writeEvents[bi]))
+    #else
+    {
+      unsigned wi;
+      {
+        CriticalSection_Enter(&mtc->cs);
+        wi = mtc->writeIndex;
+        if (wi == bi)
+          mtc->writeIndex = (unsigned)(int)-1;
+        else
+          mtc->ReadyBlocks[bi] = True;
+        CriticalSection_Leave(&mtc->cs);
+      }
+
+      if (wi != bi)
+      {
+        if (res != SZ_OK || finished)
+          return 0;
+        continue;
+      }
+
+      if (mtc->writeRes != SZ_OK)
+        res = mtc->writeRes;
+
+      for (;;)
+      {
+        if (res == SZ_OK && bufIndex != (unsigned)(int)-1)
+        {
+          res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex);
+          if (res != SZ_OK)
+          {
+            mtc->writeRes = res;
+            MtProgress_SetError(&mtc->mtProgress, res);
+          }
+        }
+
+        if (++wi >= mtc->numBlocksMax)
+          wi = 0;
+        {
+          Bool isReady;
+
+          CriticalSection_Enter(&mtc->cs);
+          
+          if (bufIndex != (unsigned)(int)-1)
+          {
+            mtc->freeBlockList[bufIndex] = mtc->freeBlockHead;
+            mtc->freeBlockHead = bufIndex;
+          }
+          
+          isReady = mtc->ReadyBlocks[wi];
+          
+          if (isReady)
+            mtc->ReadyBlocks[wi] = False;
+          else
+            mtc->writeIndex = wi;
+          
+          CriticalSection_Leave(&mtc->cs);
+
+          RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore))
+
+          if (!isReady)
+            break;
+        }
+
+        {
+          CMtCoderBlock *block = &mtc->blocks[wi];
+          if (res == SZ_OK && block->res != SZ_OK)
+            res = block->res;
+          bufIndex = block->bufIndex;
+          finished = block->finished;
+        }
+      }
+    }
+    #endif
+      
+    if (finished || res != SZ_OK)
       return 0;
   }
 }
 
-void MtCoder_Construct(CMtCoder* p)
+
+static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
+{
+  CMtCoderThread *t = (CMtCoderThread *)pp;
+  for (;;)
+  {
+    if (Event_Wait(&t->startEvent) != 0)
+      return SZ_ERROR_THREAD;
+    if (t->stop)
+      return 0;
+    {
+      SRes res = ThreadFunc2(t);
+      CMtCoder *mtc = t->mtCoder;
+      if (res != SZ_OK)
+      {
+        MtProgress_SetError(&mtc->mtProgress, res);
+      }
+      
+      #ifndef MTCODER__USE_WRITE_THREAD
+      {
+        unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads);
+        if (numFinished == mtc->numStartedThreads)
+          if (Event_Set(&mtc->finishedEvent) != 0)
+            return SZ_ERROR_THREAD;
+      }
+      #endif
+    }
+  }
+}
+
+
+
+void MtCoder_Construct(CMtCoder *p)
 {
   unsigned i;
-  p->alloc = 0;
-  for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
+  
+  p->blockSize = 0;
+  p->numThreadsMax = 0;
+  p->expectedDataSize = (UInt64)(Int64)-1;
+
+  p->inStream = NULL;
+  p->inData = NULL;
+  p->inDataSize = 0;
+
+  p->progress = NULL;
+  p->allocBig = NULL;
+
+  p->mtCallback = NULL;
+  p->mtCallbackObject = NULL;
+
+  p->allocatedBufsSize = 0;
+
+  Event_Construct(&p->readEvent);
+  Semaphore_Construct(&p->blocksSemaphore);
+
+  for (i = 0; i < MTCODER__THREADS_MAX; i++)
   {
-    CMtThread *t = &p->threads[i];
+    CMtCoderThread *t = &p->threads[i];
+    t->mtCoder = p;
     t->index = i;
-    CMtThread_Construct(t, p);
+    t->inBuf = NULL;
+    t->stop = False;
+    Event_Construct(&t->startEvent);
+    Thread_Construct(&t->thread);
   }
+
+  #ifdef MTCODER__USE_WRITE_THREAD
+    for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
+      Event_Construct(&p->writeEvents[i]);
+  #else
+    Event_Construct(&p->finishedEvent);
+  #endif
+
   CriticalSection_Init(&p->cs);
   CriticalSection_Init(&p->mtProgress.cs);
 }
 
-void MtCoder_Destruct(CMtCoder* p)
+
+
+
+static void MtCoder_Free(CMtCoder *p)
 {
   unsigned i;
-  for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
-    CMtThread_Destruct(&p->threads[i]);
+
+  /*
+  p->stopReading = True;
+  if (Event_IsCreated(&p->readEvent))
+    Event_Set(&p->readEvent);
+  */
+
+  for (i = 0; i < MTCODER__THREADS_MAX; i++)
+    MtCoderThread_Destruct(&p->threads[i]);
+
+  Event_Close(&p->readEvent);
+  Semaphore_Close(&p->blocksSemaphore);
+
+  #ifdef MTCODER__USE_WRITE_THREAD
+    for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
+      Event_Close(&p->writeEvents[i]);
+  #else
+    Event_Close(&p->finishedEvent);
+  #endif
+}
+
+
+void MtCoder_Destruct(CMtCoder *p)
+{
+  MtCoder_Free(p);
+
   CriticalSection_Delete(&p->cs);
   CriticalSection_Delete(&p->mtProgress.cs);
 }
 
+
 SRes MtCoder_Code(CMtCoder *p)
 {
-  unsigned i, numThreads = p->numThreads;
+  unsigned numThreads = p->numThreadsMax;
+  unsigned numBlocksMax;
+  unsigned i;
   SRes res = SZ_OK;
-  p->res = SZ_OK;
+
+  if (numThreads > MTCODER__THREADS_MAX)
+    numThreads = MTCODER__THREADS_MAX;
+  numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads);
+  
+  if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++;
+  if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++;
+  if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++;
+
+  if (numBlocksMax > MTCODER__BLOCKS_MAX)
+    numBlocksMax = MTCODER__BLOCKS_MAX;
+
+  if (p->blockSize != p->allocatedBufsSize)
+  {
+    for (i = 0; i < MTCODER__THREADS_MAX; i++)
+    {
+      CMtCoderThread *t = &p->threads[i];
+      if (t->inBuf)
+      {
+        ISzAlloc_Free(p->allocBig, t->inBuf);
+        t->inBuf = NULL;
+      }
+    }
+    p->allocatedBufsSize = p->blockSize;
+  }
+
+  p->readRes = SZ_OK;
 
   MtProgress_Init(&p->mtProgress, p->progress);
 
-  for (i = 0; i < numThreads; i++)
+  #ifdef MTCODER__USE_WRITE_THREAD
+    for (i = 0; i < numBlocksMax; i++)
+    {
+      RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i]));
+    }
+  #else
+    RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent));
+  #endif
+
   {
-    RINOK(CMtThread_Prepare(&p->threads[i]));
+    RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent));
+
+    if (Semaphore_IsCreated(&p->blocksSemaphore))
+    {
+      RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore));
+    }
+    RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax));
   }
 
-  for (i = 0; i < numThreads; i++)
+  for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++)
+    p->freeBlockList[i] = i + 1;
+  p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1;
+  p->freeBlockHead = 0;
+
+  p->readProcessed = 0;
+  p->blockIndex = 0;
+  p->numBlocksMax = numBlocksMax;
+  p->stopReading = False;
+
+  #ifndef MTCODER__USE_WRITE_THREAD
+    p->writeIndex = 0;
+    p->writeRes = SZ_OK;
+    for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
+      p->ReadyBlocks[i] = False;
+    p->numFinishedThreads = 0;
+  #endif
+
+  p->numStartedThreadsLimit = numThreads;
+  p->numStartedThreads = 0;
+
+  // for (i = 0; i < numThreads; i++)
   {
-    CMtThread *t = &p->threads[i];
-    CLoopThread *lt = &t->thread;
+    CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++];
+    RINOK(MtCoderThread_CreateAndStart(nextThread));
+  }
 
-    if (!Thread_WasCreated(&lt->thread))
+  RINOK_THREAD(Event_Set(&p->readEvent))
+
+  #ifdef MTCODER__USE_WRITE_THREAD
+  {
+    unsigned bi = 0;
+
+    for (;; bi++)
     {
-      lt->func = ThreadFunc;
-      lt->param = t;
+      if (bi >= numBlocksMax)
+        bi = 0;
+
+      RINOK_THREAD(Event_Wait(&p->writeEvents[bi]))
 
-      if (LoopThread_Create(lt) != SZ_OK)
       {
-        res = SZ_ERROR_THREAD;
-        break;
+        const CMtCoderBlock *block = &p->blocks[bi];
+        unsigned bufIndex = block->bufIndex;
+        Bool finished = block->finished;
+        if (res == SZ_OK && block->res != SZ_OK)
+          res = block->res;
+
+        if (bufIndex != (unsigned)(int)-1)
+        {
+          if (res == SZ_OK)
+          {
+            res = p->mtCallback->Write(p->mtCallbackObject, bufIndex);
+            if (res != SZ_OK)
+              MtProgress_SetError(&p->mtProgress, res);
+          }
+          
+          CriticalSection_Enter(&p->cs);
+          {
+            p->freeBlockList[bufIndex] = p->freeBlockHead;
+            p->freeBlockHead = bufIndex;
+          }
+          CriticalSection_Leave(&p->cs);
+        }
+        
+        RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore))
+
+        if (finished)
+          break;
       }
     }
   }
+  #else
+  {
+    WRes wres = Event_Wait(&p->finishedEvent);
+    res = MY_SRes_HRESULT_FROM_WRes(wres);
+  }
+  #endif
 
   if (res == SZ_OK)
-  {
-    unsigned j;
-    for (i = 0; i < numThreads; i++)
-    {
-      CMtThread *t = &p->threads[i];
-      if (LoopThread_StartSubThread(&t->thread) != SZ_OK)
-      {
-        res = SZ_ERROR_THREAD;
-        p->threads[0].stopReading = True;
-        break;
-      }
-    }
+    res = p->readRes;
 
-    Event_Set(&p->threads[0].canWrite);
-    Event_Set(&p->threads[0].canRead);
+  if (res == SZ_OK)
+    res = p->mtProgress.res;
 
-    for (j = 0; j < i; j++)
-      LoopThread_WaitSubThread(&p->threads[j].thread);
-  }
+  #ifndef MTCODER__USE_WRITE_THREAD
+    if (res == SZ_OK)
+      res = p->writeRes;
+  #endif
 
-  for (i = 0; i < numThreads; i++)
-    CMtThread_CloseEvents(&p->threads[i]);
-  return (res == SZ_OK) ? p->res : res;
+  if (res != SZ_OK)
+    MtCoder_Free(p);
+  return res;
 }
+
+#endif

C/MtCoder.h

@@ -1,102 +1,141 @@
 /* MtCoder.h -- Multi-thread Coder
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-21 : Igor Pavlov : Public domain */
 
 #ifndef __MT_CODER_H
 #define __MT_CODER_H
 
-#include "Threads.h"
+#include "MtDec.h"
 
 EXTERN_C_BEGIN
 
-typedef struct
-{
-  CThread thread;
-  CAutoResetEvent startEvent;
-  CAutoResetEvent finishedEvent;
-  int stop;
-  
-  THREAD_FUNC_TYPE func;
-  LPVOID param;
-  THREAD_FUNC_RET_TYPE res;
-} CLoopThread;
-
-void LoopThread_Construct(CLoopThread *p);
-void LoopThread_Close(CLoopThread *p);
-WRes LoopThread_Create(CLoopThread *p);
-WRes LoopThread_StopAndWait(CLoopThread *p);
-WRes LoopThread_StartSubThread(CLoopThread *p);
-WRes LoopThread_WaitSubThread(CLoopThread *p);
+/*
+  if (    defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream
+  if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream
+*/
+/* #define MTCODER__USE_WRITE_THREAD */
 
 #ifndef _7ZIP_ST
-#define NUM_MT_CODER_THREADS_MAX 32
+  #define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1)
+  #define MTCODER__THREADS_MAX 64
+  #define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3)
 #else
-#define NUM_MT_CODER_THREADS_MAX 1
+  #define MTCODER__THREADS_MAX 1
+  #define MTCODER__BLOCKS_MAX 1
 #endif
 
+
+#ifndef _7ZIP_ST
+
+
 typedef struct
 {
-  UInt64 totalInSize;
-  UInt64 totalOutSize;
-  ICompressProgress *progress;
-  SRes res;
-  CCriticalSection cs;
-  UInt64 inSizes[NUM_MT_CODER_THREADS_MAX];
-  UInt64 outSizes[NUM_MT_CODER_THREADS_MAX];
-} CMtProgress;
+  ICompressProgress vt;
+  CMtProgress *mtProgress;
+  UInt64 inSize;
+  UInt64 outSize;
+} CMtProgressThunk;
+
+void MtProgressThunk_CreateVTable(CMtProgressThunk *p);
+    
+#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; }
 
-SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize);
 
 struct _CMtCoder;
 
+
 typedef struct
 {
   struct _CMtCoder *mtCoder;
-  Byte *outBuf;
-  size_t outBufSize;
-  Byte *inBuf;
-  size_t inBufSize;
   unsigned index;
-  CLoopThread thread;
+  int stop;
+  Byte *inBuf;
 
-  Bool stopReading;
-  Bool stopWriting;
-  CAutoResetEvent canRead;
-  CAutoResetEvent canWrite;
-} CMtThread;
+  CAutoResetEvent startEvent;
+  CThread thread;
+} CMtCoderThread;
 
 
-typedef struct IMtCoderCallback IMtCoderCallback;
-struct IMtCoderCallback
+typedef struct
 {
-  SRes (*Code)(const IMtCoderCallback *p, unsigned index, Byte *dest, size_t *destSize,
+  SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex,
       const Byte *src, size_t srcSize, int finished);
-};
-#define IMtCoderCallback_Code(p, index, dest, destSize, src, srcSize, finished) (p)->Code(p, index, dest, destSize, src, srcSize, finished)
+  SRes (*Write)(void *p, unsigned outBufIndex);
+} IMtCoderCallback2;
+
+
+typedef struct
+{
+  SRes res;
+  unsigned bufIndex;
+  Bool finished;
+} CMtCoderBlock;
 
 
 typedef struct _CMtCoder
 {
-  size_t blockSize;
-  size_t destBlockSize;
-  unsigned numThreads;
+  /* input variables */
   
-  ISeqInStream *inStream;
-  ISeqOutStream *outStream;
-  ICompressProgress *progress;
-  ISzAllocPtr alloc;
+  size_t blockSize;        /* size of input block */
+  unsigned numThreadsMax;
+  UInt64 expectedDataSize;
+
+  ISeqInStream *inStream;
+  const Byte *inData;
+  size_t inDataSize;
+
+  ICompressProgress *progress;
+  ISzAllocPtr allocBig;
+
+  IMtCoderCallback2 *mtCallback;
+  void *mtCallbackObject;
+
+  
+  /* internal variables */
+  
+  size_t allocatedBufsSize;
+
+  CAutoResetEvent readEvent;
+  CSemaphore blocksSemaphore;
+
+  Bool stopReading;
+  SRes readRes;
+
+  #ifdef MTCODER__USE_WRITE_THREAD
+    CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX];
+  #else
+    CAutoResetEvent finishedEvent;
+    SRes writeRes;
+    unsigned writeIndex;
+    Byte ReadyBlocks[MTCODER__BLOCKS_MAX];
+    LONG numFinishedThreads;
+  #endif
+
+  unsigned numStartedThreadsLimit;
+  unsigned numStartedThreads;
+
+  unsigned numBlocksMax;
+  unsigned blockIndex;
+  UInt64 readProcessed;
 
-  IMtCoderCallback *mtCallback;
   CCriticalSection cs;
-  SRes res;
+
+  unsigned freeBlockHead;
+  unsigned freeBlockList[MTCODER__BLOCKS_MAX];
 
   CMtProgress mtProgress;
-  CMtThread threads[NUM_MT_CODER_THREADS_MAX];
+  CMtCoderBlock blocks[MTCODER__BLOCKS_MAX];
+  CMtCoderThread threads[MTCODER__THREADS_MAX];
 } CMtCoder;
 
-void MtCoder_Construct(CMtCoder* p);
-void MtCoder_Destruct(CMtCoder* p);
+
+void MtCoder_Construct(CMtCoder *p);
+void MtCoder_Destruct(CMtCoder *p);
 SRes MtCoder_Code(CMtCoder *p);
 
+
+#endif
+
+
 EXTERN_C_END
 
 #endif

C/MtDec.h

@@ -0,0 +1,201 @@
+/* MtDec.h -- Multi-thread Decoder
+2018-03-02 : Igor Pavlov : Public domain */
+
+#ifndef __MT_DEC_H
+#define __MT_DEC_H
+
+#include "7zTypes.h"
+
+#ifndef _7ZIP_ST
+#include "Threads.h"
+#endif
+
+EXTERN_C_BEGIN
+
+#ifndef _7ZIP_ST
+
+#ifndef _7ZIP_ST
+  #define MTDEC__THREADS_MAX 32
+#else
+  #define MTDEC__THREADS_MAX 1
+#endif
+
+
+typedef struct
+{
+  ICompressProgress *progress;
+  SRes res;
+  UInt64 totalInSize;
+  UInt64 totalOutSize;
+  CCriticalSection cs;
+} CMtProgress;
+
+void MtProgress_Init(CMtProgress *p, ICompressProgress *progress);
+SRes MtProgress_Progress_ST(CMtProgress *p);
+SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize);
+SRes MtProgress_GetError(CMtProgress *p);
+void MtProgress_SetError(CMtProgress *p, SRes res);
+
+struct _CMtDec;
+
+typedef struct
+{
+  struct _CMtDec *mtDec;
+  unsigned index;
+  void *inBuf;
+
+  size_t inDataSize_Start; // size of input data in start block
+  UInt64 inDataSize;       // total size of input data in all blocks
+
+  CThread thread;
+  CAutoResetEvent canRead;
+  CAutoResetEvent canWrite;
+  void  *allocaPtr;
+} CMtDecThread;
+
+void MtDecThread_FreeInBufs(CMtDecThread *t);
+
+
+typedef enum
+{
+  MTDEC_PARSE_CONTINUE, // continue this block with more input data
+  MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread
+  MTDEC_PARSE_NEW,      // new block
+  MTDEC_PARSE_END       // end of block threading. But we still can return to threading after Write(&needContinue)
+} EMtDecParseState;
+
+typedef struct
+{
+  // in
+  int startCall;
+  const Byte *src;
+  size_t srcSize;
+      // in  : (srcSize == 0) is allowed
+      // out : it's allowed to return less that actually was used ?
+  int srcFinished;
+
+  // out
+  EMtDecParseState state;
+  Bool canCreateNewThread;
+  UInt64 outPos; // check it (size_t)
+} CMtDecCallbackInfo;
+
+
+typedef struct
+{
+  void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci);
+  
+  // PreCode() and Code():
+  // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks
+  SRes (*PreCode)(void *p, unsigned coderIndex);
+  SRes (*Code)(void *p, unsigned coderIndex,
+      const Byte *src, size_t srcSize, int srcFinished,
+      UInt64 *inCodePos, UInt64 *outCodePos, int *stop);
+  // stop - means stop another Code calls
+
+
+  /* Write() must be called, if Parse() was called
+      set (needWrite) if
+      {
+         && (was not interrupted by progress)
+         && (was not interrupted in previous block)
+      }
+
+    out:
+      if (*needContinue), decoder still need to continue decoding with new iteration,
+         even after MTDEC_PARSE_END
+      if (*canRecode), we didn't flush current block data, so we still can decode current block later.
+  */
+  SRes (*Write)(void *p, unsigned coderIndex,
+      Bool needWriteToStream,
+      const Byte *src, size_t srcSize,
+      // int srcFinished,
+      Bool *needContinue,
+      Bool *canRecode);
+} IMtDecCallback;
+
+
+
+typedef struct _CMtDec
+{
+  /* input variables */
+  
+  size_t inBufSize;        /* size of input block */
+  unsigned numThreadsMax;
+  // size_t inBlockMax;
+  unsigned numThreadsMax_2;
+
+  ISeqInStream *inStream;
+  // const Byte *inData;
+  // size_t inDataSize;
+
+  ICompressProgress *progress;
+  ISzAllocPtr alloc;
+
+  IMtDecCallback *mtCallback;
+  void *mtCallbackObject;
+
+  
+  /* internal variables */
+  
+  size_t allocatedBufsSize;
+
+  Bool exitThread;
+  WRes exitThreadWRes;
+
+  UInt64 blockIndex;
+  Bool isAllocError;
+  Bool overflow;
+  SRes threadingErrorSRes;
+
+  Bool needContinue;
+
+  // CAutoResetEvent finishedEvent;
+
+  SRes readRes;
+  SRes codeRes;
+
+  Bool wasInterrupted;
+
+  unsigned numStartedThreads_Limit;
+  unsigned numStartedThreads;
+
+  Byte *crossBlock;
+  size_t crossStart;
+  size_t crossEnd;
+  UInt64 readProcessed;
+  Bool readWasFinished;
+  UInt64 inProcessed;
+
+  unsigned filledThreadStart;
+  unsigned numFilledThreads;
+
+  #ifndef _7ZIP_ST
+  Bool needInterrupt;
+  UInt64 interruptIndex;
+  CMtProgress mtProgress;
+  CMtDecThread threads[MTDEC__THREADS_MAX];
+  #endif
+} CMtDec;
+
+
+void MtDec_Construct(CMtDec *p);
+void MtDec_Destruct(CMtDec *p);
+
+/*
+MtDec_Code() returns:
+  SZ_OK - in most cases
+  MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function
+*/
+  
+SRes MtDec_Code(CMtDec *p);
+Byte *MtDec_GetCrossBuff(CMtDec *p);
+
+int MtDec_PrepareRead(CMtDec *p);
+const Byte *MtDec_Read(CMtDec *p, size_t *inLim);
+
+#endif
+
+EXTERN_C_END
+
+#endif

C/Threads.c

@@ -1,5 +1,5 @@
 /* Threads.c -- multithreading library
-2014-09-21 : Igor Pavlov : Public domain */
+2017-06-26 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -12,18 +12,20 @@
 static WRes GetError()
 {
   DWORD res = GetLastError();
-  return (res) ? (WRes)(res) : 1;
+  return res ? (WRes)res : 1;
 }
 
-WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); }
-WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
+static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); }
+static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
 
 WRes HandlePtr_Close(HANDLE *p)
 {
   if (*p != NULL)
+  {
     if (!CloseHandle(*p))
       return GetError();
-  *p = NULL;
+    *p = NULL;
+  }
   return 0;
 }
 
@@ -49,7 +51,7 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
   return HandleToWRes(*p);
 }
 
-WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
+static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
 {
   *p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL);
   return HandleToWRes(*p);

C/Threads.h

@@ -1,5 +1,5 @@
 /* Threads.h -- multithreading library
-2013-11-12 : Igor Pavlov : Public domain */
+2017-06-18 : Igor Pavlov : Public domain */
 
 #ifndef __7Z_THREADS_H
 #define __7Z_THREADS_H
@@ -49,7 +49,8 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled);
 WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p);
 
 typedef HANDLE CSemaphore;
-#define Semaphore_Construct(p) (*p) = NULL
+#define Semaphore_Construct(p) *(p) = NULL
+#define Semaphore_IsCreated(p) (*(p) != NULL)
 #define Semaphore_Close(p) HandlePtr_Close(p)
 #define Semaphore_Wait(p) Handle_WaitObject(*(p))
 WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount);

C/Util/7z/7zMain.c

@@ -1,5 +1,5 @@
 /* 7zMain.c - Test application for 7z Decoder
-2017-04-05 : Igor Pavlov : Public domain */
+2017-08-26 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -616,7 +616,14 @@ int MY_CDECL main(int numargs, char *args[])
           
           #ifdef USE_WINDOWS_FILE
           if (SzBitWithVals_Check(&db.Attribs, i))
-            SetFileAttributesW(destPath, db.Attribs.Vals[i]);
+          {
+            UInt32 attrib = db.Attribs.Vals[i];
+            /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker.
+               We remove posix bits, if we detect posix mode field */
+            if ((attrib & 0xF0000000) != 0)
+              attrib &= 0x7FFF;
+            SetFileAttributesW(destPath, attrib);
+          }
           #endif
         }
         PrintLF();

C/Util/7zipInstall/7zipInstall.c

@@ -1,5 +1,5 @@
 /* 7zipInstall.c - 7-Zip Installer
-2017-04-04 : Igor Pavlov : Public domain */
+2017-08-28 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -154,7 +154,7 @@ static WRes MyCreateDir(LPCWSTR name)
 }
 
 #define IS_SEPAR(c) (c == WCHAR_PATH_SEPARATOR)
-#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z')
+#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
 #define IS_DRIVE_PATH(s) (IS_LETTER_CHAR(s[0]) && s[1] == ':' && IS_SEPAR(s[2]))
 
 static int ReverseFind_PathSepar(const wchar_t *s)

C/Util/7zipUninstall/7zipUninstall.c

@@ -1,5 +1,5 @@
 /* 7zipUninstall.c - 7-Zip Uninstaller
-2017-04-04 : Igor Pavlov : Public domain */
+2018-03-01 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -546,7 +546,7 @@ static BOOL RemoveDir()
 #define k_Lang L"Lang"
 
 // NUM_LANG_TXT_FILES files are placed before en.ttt
-#define NUM_LANG_TXT_FILES 87
+#define NUM_LANG_TXT_FILES 88
 
 #ifdef _64BIT_INSTALLER
   #define NUM_EXTRA_FILES_64BIT 1
@@ -558,7 +558,7 @@ static BOOL RemoveDir()
 
 static const char * const k_Names =
   "af an ar ast az ba be bg bn br ca co cs cy da de el eo es et eu ext"
-  " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kk ko ku ku-ckb ky"
+  " fa fi fr fur fy ga gl gu he hi hr hu hy id io is it ja ka kaa kab kk ko ku ku-ckb ky"
   " lij lt lv mk mn mng mng2 mr ms nb ne nl nn pa-in pl ps pt pt-br ro ru"
   " sa si sk sl sq sr-spc sr-spl sv ta th tr tt ug uk uz va vi yo zh-cn zh-tw"
   " en.ttt"

C/Util/Lzma/LzmaUtil.dsp

@@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\7zTypes.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\7zVersion.h
 # End Source File
 # Begin Source File
@@ -160,9 +164,5 @@ SOURCE=..\..\Threads.c
 
 SOURCE=..\..\Threads.h
 # End Source File
-# Begin Source File
-
-SOURCE=..\..\Types.h
-# End Source File
 # End Target
 # End Project

C/Xz.c

@@ -1,5 +1,5 @@
 /* Xz.c - Xz
-2017-04-03 : Igor Pavlov : Public domain */
+2017-05-12 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -9,7 +9,7 @@
 #include "XzCrc64.h"
 
 const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 };
-const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' };
+/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */
 
 unsigned Xz_WriteVarInt(Byte *buf, UInt64 v)
 {
@@ -26,16 +26,16 @@ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v)
 
 void Xz_Construct(CXzStream *p)
 {
-  p->numBlocks = p->numBlocksAllocated = 0;
-  p->blocks = 0;
+  p->numBlocks = 0;
+  p->blocks = NULL;
   p->flags = 0;
 }
 
 void Xz_Free(CXzStream *p, ISzAllocPtr alloc)
 {
   ISzAlloc_Free(alloc, p->blocks);
-  p->numBlocks = p->numBlocksAllocated = 0;
-  p->blocks = 0;
+  p->numBlocks = 0;
+  p->blocks = NULL;
 }
 
 unsigned XzFlags_GetCheckSize(CXzStreamFlags f)

C/Xz.h

@@ -1,5 +1,5 @@
 /* Xz.h - Xz interface
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-28 : Igor Pavlov : Public domain */
 
 #ifndef __XZ_H
 #define __XZ_H
@@ -50,6 +50,7 @@ typedef struct
 #define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1)
 #define XzBlock_HasPackSize(p)   (((p)->flags & XZ_BF_PACK_SIZE) != 0)
 #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0)
+#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0)
 
 SRes XzBlock_Parse(CXzBlock *p, const Byte *header);
 SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes);
@@ -60,7 +61,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
 #define XZ_FOOTER_SIG_SIZE 2
 
 extern const Byte XZ_SIG[XZ_SIG_SIZE];
+
+/*
 extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE];
+*/
+
+#define XZ_FOOTER_SIG_0 'Y'
+#define XZ_FOOTER_SIG_1 'Z'
 
 #define XZ_STREAM_FLAGS_SIZE 2
 #define XZ_STREAM_CRC_SIZE 4
@@ -106,7 +113,6 @@ typedef struct
 {
   CXzStreamFlags flags;
   size_t numBlocks;
-  size_t numBlocksAllocated;
   CXzBlockSizes *blocks;
   UInt64 startOffset;
 } CXzStream;
@@ -133,6 +139,9 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICom
 UInt64 Xzs_GetNumBlocks(const CXzs *p);
 UInt64 Xzs_GetUnpackSize(const CXzs *p);
 
+
+// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder
+
 typedef enum
 {
   CODER_STATUS_NOT_SPECIFIED,               /* use main error code instead */
@@ -141,22 +150,31 @@ typedef enum
   CODER_STATUS_NEEDS_MORE_INPUT             /* you must provide more input bytes */
 } ECoderStatus;
 
+
+// ECoderFinishMode values are identical to ELzmaFinishMode
+
 typedef enum
 {
   CODER_FINISH_ANY,   /* finish at any point */
   CODER_FINISH_END    /* block must be finished at the end */
 } ECoderFinishMode;
 
+
 typedef struct _IStateCoder
 {
   void *p;
   void (*Free)(void *p, ISzAllocPtr alloc);
   SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc);
   void (*Init)(void *p);
-  SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
-      int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished);
+  SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen,
+      int srcWasFinished, ECoderFinishMode finishMode,
+      // int *wasFinished,
+      ECoderStatus *status);
+  SizeT (*Filter)(void *p, Byte *data, SizeT size);
 } IStateCoder;
 
+
+
 #define MIXCODER_NUM_FILTERS_MAX 4
 
 typedef struct
@@ -164,20 +182,23 @@ typedef struct
   ISzAllocPtr alloc;
   Byte *buf;
   unsigned numCoders;
+
+  Byte *outBuf;
+  size_t outBufSize;
+  size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode)
+  Bool wasFinished;
+  SRes res;
+  ECoderStatus status;
+  // Bool SingleBufMode;
+  
   int finished[MIXCODER_NUM_FILTERS_MAX - 1];
   size_t pos[MIXCODER_NUM_FILTERS_MAX - 1];
   size_t size[MIXCODER_NUM_FILTERS_MAX - 1];
   UInt64 ids[MIXCODER_NUM_FILTERS_MAX];
+  SRes results[MIXCODER_NUM_FILTERS_MAX];
   IStateCoder coders[MIXCODER_NUM_FILTERS_MAX];
 } CMixCoder;
 
-void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc);
-void MixCoder_Free(CMixCoder *p);
-void MixCoder_Init(CMixCoder *p);
-SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId);
-SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
-    const Byte *src, SizeT *srcLen, int srcWasFinished,
-    ECoderFinishMode finishMode, ECoderStatus *status);
 
 typedef enum
 {
@@ -191,6 +212,7 @@ typedef enum
   XZ_STATE_BLOCK_FOOTER
 } EXzState;
 
+
 typedef struct
 {
   EXzState state;
@@ -204,7 +226,7 @@ typedef struct
   UInt64 packSize;
   UInt64 unpackSize;
 
-  UInt64 numBlocks;
+  UInt64 numBlocks; // number of finished blocks in current stream
   UInt64 indexSize;
   UInt64 indexPos;
   UInt64 padSize;
@@ -218,14 +240,65 @@ typedef struct
   CXzBlock block;
   CXzCheck check;
   CSha256 sha;
+
+  Bool parseMode;
+  Bool headerParsedOk;
+  Bool decodeToStreamSignature;
+  unsigned decodeOnlyOneBlock;
+
+  Byte *outBuf;
+  size_t outBufSize;
+  size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked
+
   Byte shaDigest[SHA256_DIGEST_SIZE];
   Byte buf[XZ_BLOCK_HEADER_SIZE_MAX];
 } CXzUnpacker;
 
+/* alloc : aligned for cache line allocation is better */
 void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc);
 void XzUnpacker_Init(CXzUnpacker *p);
+void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize);
 void XzUnpacker_Free(CXzUnpacker *p);
 
+/*
+  XzUnpacker
+  The sequence for decoding functions:
+  {
+    XzUnpacker_Construct()
+    [Decoding_Calls]
+    XzUnpacker_Free()
+  }
+
+  [Decoding_Calls]
+
+  There are 3 types of interfaces for [Decoding_Calls] calls:
+
+  Interface-1 : Partial output buffers:
+    {
+      XzUnpacker_Init()
+      for()
+        XzUnpacker_Code();
+    }
+    
+  Interface-2 : Direct output buffer:
+    Use it, if you know exact size of decoded data, and you need
+    whole xz unpacked data in one output buffer.
+    xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode.
+    {
+      XzUnpacker_Init()
+      XzUnpacker_SetOutBufMode(); // to set output buffer and size
+      for()
+        XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code()
+    }
+
+  Interface-3 : Direct output buffer : One call full decoding
+    It unpacks whole input buffer to output buffer in one call.
+    It uses Interface-2 internally.
+    {
+      XzUnpacker_CodeFull()
+    }
+*/
+
 /*
 finishMode:
   It has meaning only if the decoding reaches output limit (*destLen).
@@ -255,20 +328,132 @@ Returns:
 
 
 SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
-    const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode,
-    ECoderStatus *status);
+    const Byte *src, SizeT *srcLen, int srcFinished,
+    ECoderFinishMode finishMode, ECoderStatus *status);
 
-Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p);
+SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen,
+    const Byte *src, SizeT *srcLen,
+    ECoderFinishMode finishMode, ECoderStatus *status);
+
+Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p);
 
 /*
-Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of
-xz stream in two cases:
-XzUnpacker_Code() returns:
+XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes,
+ if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state.
+These bytes can be some bytes after xz archive, or
+it can be start of new xz stream.
+ 
+Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of
+xz stream in two cases, if XzUnpacker_Code() returns:
   res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT
   res == SZ_ERROR_NO_ARCHIVE
 */
 
-UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p);
+UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p);
+
+
+/*
+  for random block decoding:
+    XzUnpacker_Init();
+    set CXzUnpacker::streamFlags
+    XzUnpacker_PrepareToRandomBlockDecoding()
+    loop
+    {
+      XzUnpacker_Code()
+      XzUnpacker_IsBlockFinished()
+    }
+*/
+
+void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p);
+Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p);
+
+#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags))
+
+
+
+/* ---------- Multi Threading Decoding ---------- */
+
+
+typedef struct
+{
+  size_t inBufSize_ST;
+  size_t outStep_ST;
+  Bool ignoreErrors;
+  
+  #ifndef _7ZIP_ST
+  unsigned numThreads;
+  size_t inBufSize_MT;
+  size_t memUseMax;
+  #endif
+} CXzDecMtProps;
+
+void XzDecMtProps_Init(CXzDecMtProps *p);
+
+
+typedef void * CXzDecMtHandle;
+
+/*
+  alloc    : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc).
+  allocMid : for big allocations, aligned allocation is better
+*/
+
+CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid);
+void XzDecMt_Destroy(CXzDecMtHandle p);
+
+
+typedef struct
+{
+  Byte UnpackSize_Defined;
+  Byte NumStreams_Defined;
+  Byte NumBlocks_Defined;
+
+  Byte DataAfterEnd;
+  Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data
+
+  UInt64 InSize;  // pack size processed
+  UInt64 OutSize;
+
+  UInt64 NumStreams;
+  UInt64 NumBlocks;
+
+  SRes DecodeRes;
+  SRes ReadRes;
+  SRes ProgressRes;
+  SRes CombinedRes;
+  SRes CombinedRes_Type;
+
+} CXzStatInfo;
+
+void XzStatInfo_Clear(CXzStatInfo *p);
+
+/*
+XzDecMt_Decode()
+SRes:
+  SZ_OK               - OK
+  SZ_ERROR_MEM        - Memory allocation error
+  SZ_ERROR_NO_ARCHIVE - is not xz archive
+  SZ_ERROR_ARCHIVE    - Headers error
+  SZ_ERROR_DATA       - Data Error
+  SZ_ERROR_CRC        - CRC Error
+  SZ_ERROR_INPUT_EOF  - it needs more input data
+  SZ_ERROR_WRITE      - ISeqOutStream error
+  (SZ_ERROR_READ)     - ISeqInStream errors
+  (SZ_ERROR_PROGRESS) - ICompressProgress errors
+  // SZ_ERROR_THREAD     - error in multi-threading functions
+  MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function
+*/
+
+SRes XzDecMt_Decode(CXzDecMtHandle p,
+    const CXzDecMtProps *props,
+    const UInt64 *outDataSize, // NULL means undefined
+    int finishMode,            // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished
+    ISeqOutStream *outStream,
+    // Byte *outBuf, size_t *outBufSize,
+    ISeqInStream *inStream,
+    // const Byte *inData, size_t inDataSize,
+    CXzStatInfo *stat,
+    int *isMT,                 // 0 means that ST (Single-Thread) version was used
+    ICompressProgress *progress);
 
 EXTERN_C_END
 

C/XzCrc64.c

@@ -1,5 +1,5 @@
 /* XzCrc64.c -- CRC64 calculation
-2017-04-03 : Igor Pavlov : Public domain */
+2017-05-23 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -9,9 +9,9 @@
 #define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42)
 
 #ifdef MY_CPU_LE
-  #define CRC_NUM_TABLES 4
+  #define CRC64_NUM_TABLES 4
 #else
-  #define CRC_NUM_TABLES 5
+  #define CRC64_NUM_TABLES 5
   #define CRC_UINT64_SWAP(v) \
       ((v >> 56) \
     | ((v >> 40) & ((UInt64)0xFF <<  8)) \
@@ -29,10 +29,10 @@
   UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table);
 #endif
 
-typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table);
+typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table);
 
-static CRC_FUNC g_Crc64Update;
-UInt64 g_Crc64Table[256 * CRC_NUM_TABLES];
+static CRC64_FUNC g_Crc64Update;
+UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES];
 
 UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size)
 {
@@ -55,7 +55,7 @@ void MY_FAST_CALL Crc64GenerateTable()
       r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1)));
     g_Crc64Table[i] = r;
   }
-  for (i = 256; i < 256 * CRC_NUM_TABLES; i++)
+  for (i = 256; i < 256 * CRC64_NUM_TABLES; i++)
   {
     UInt64 r = g_Crc64Table[(size_t)i - 256];
     g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8);
@@ -74,7 +74,7 @@ void MY_FAST_CALL Crc64GenerateTable()
     else
     #endif
     {
-      for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--)
+      for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--)
       {
         UInt64 x = g_Crc64Table[(size_t)i - 256];
         g_Crc64Table[i] = CRC_UINT64_SWAP(x);

C/XzCrc64Opt.c

@@ -1,5 +1,5 @@
 /* XzCrc64Opt.c -- CRC64 calculation
-2017-04-03 : Igor Pavlov : Public domain */
+2017-06-30 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -7,13 +7,13 @@
 
 #ifndef MY_CPU_BE
 
-#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
+#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
 
 UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table)
 {
   const Byte *p = (const Byte *)data;
   for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++)
-    v = CRC_UPDATE_BYTE_2(v, *p);
+    v = CRC64_UPDATE_BYTE_2(v, *p);
   for (; size >= 4; size -= 4, p += 4)
   {
     UInt32 d = (UInt32)v ^ *(const UInt32 *)p;
@@ -24,7 +24,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con
         ^ (table + 0x000)[((d >> 24))];
   }
   for (; size > 0; size--, p++)
-    v = CRC_UPDATE_BYTE_2(v, *p);
+    v = CRC64_UPDATE_BYTE_2(v, *p);
   return v;
 }
 
@@ -43,7 +43,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con
     | ((v << 40) & ((UInt64)0xFF << 48)) \
     | ((v << 56)))
 
-#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8))
+#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8))
 
 UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table)
 {
@@ -51,7 +51,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size
   table += 0x100;
   v = CRC_UINT64_SWAP(v);
   for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++)
-    v = CRC_UPDATE_BYTE_2_BE(v, *p);
+    v = CRC64_UPDATE_BYTE_2_BE(v, *p);
   for (; size >= 4; size -= 4, p += 4)
   {
     UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p;
@@ -62,7 +62,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size
         ^ (table + 0x300)[((d >> 24))];
   }
   for (; size > 0; size--, p++)
-    v = CRC_UPDATE_BYTE_2_BE(v, *p);
+    v = CRC64_UPDATE_BYTE_2_BE(v, *p);
   return CRC_UINT64_SWAP(v);
 }
 

C/XzEnc.h

@@ -1,5 +1,5 @@
 /* XzEnc.h -- Xz Encode
-2011-02-07 : Igor Pavlov : Public domain */
+2017-06-27 : Igor Pavlov : Public domain */
 
 #ifndef __XZ_ENC_H
 #define __XZ_ENC_H
@@ -10,6 +10,11 @@
 
 EXTERN_C_BEGIN
 
+
+#define XZ_PROPS__BLOCK_SIZE__AUTO   LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
+#define XZ_PROPS__BLOCK_SIZE__SOLID  LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
+
+
 typedef struct
 {
   UInt32 id;
@@ -20,15 +25,31 @@ typedef struct
 
 void XzFilterProps_Init(CXzFilterProps *p);
 
+
 typedef struct
 {
-  const CLzma2EncProps *lzma2Props;
-  const CXzFilterProps *filterProps;
+  CLzma2EncProps lzma2Props;
+  CXzFilterProps filterProps;
   unsigned checkId;
+  UInt64 blockSize;
+  int numBlockThreads_Reduced;
+  int numBlockThreads_Max;
+  int numTotalThreads;
+  int forceWriteSizesInHeader;
+  UInt64 reduceSize;
 } CXzProps;
 
 void XzProps_Init(CXzProps *p);
 
+
+typedef void * CXzEncHandle;
+
+CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig);
+void XzEnc_Destroy(CXzEncHandle p);
+SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props);
+void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize);
+SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
+
 SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
     const CXzProps *props, ICompressProgress *progress);
 

C/XzIn.c

@@ -1,5 +1,5 @@
 /* XzIn.c - Xz input
-2017-04-03 : Igor Pavlov : Public domain */
+2018-02-02 : Igor Pavlov : Public domain */
 
 #include "Precomp.h"
 
@@ -9,6 +9,12 @@
 #include "CpuArch.h"
 #include "Xz.h"
 
+/*
+#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0)
+*/
+#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1)
+
+
 SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream)
 {
   Byte sig[XZ_STREAM_HEADER_SIZE];
@@ -28,7 +34,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
   unsigned headerSize;
   *headerSizeRes = 0;
   RINOK(SeqInStream_ReadByte(inStream, &header[0]));
-  headerSize = ((unsigned)header[0] << 2) + 4;
+  headerSize = (unsigned)header[0];
   if (headerSize == 0)
   {
     *headerSizeRes = 1;
@@ -37,12 +43,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
   }
 
   *isIndex = False;
+  headerSize = (headerSize << 2) + 4;
   *headerSizeRes = headerSize;
   RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1));
   return XzBlock_Parse(p, header);
 }
 
-#define ADD_SIZE_CHECH(size, val) \
+#define ADD_SIZE_CHECK(size, val) \
   { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; }
 
 UInt64 Xz_GetUnpackSize(const CXzStream *p)
@@ -50,7 +57,7 @@ UInt64 Xz_GetUnpackSize(const CXzStream *p)
   UInt64 size = 0;
   size_t i;
   for (i = 0; i < p->numBlocks; i++)
-    ADD_SIZE_CHECH(size, p->blocks[i].unpackSize);
+    ADD_SIZE_CHECK(size, p->blocks[i].unpackSize);
   return size;
 }
 
@@ -59,7 +66,7 @@ UInt64 Xz_GetPackSize(const CXzStream *p)
   UInt64 size = 0;
   size_t i;
   for (i = 0; i < p->numBlocks; i++)
-    ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3);
+    ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3);
   return size;
 }
 
@@ -96,9 +103,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt
   {
     size_t i;
     p->numBlocks = numBlocks;
-    p->numBlocksAllocated = numBlocks;
-    p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks);
-    if (p->blocks == 0)
+    p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks);
+    if (!p->blocks)
       return SZ_ERROR_MEM;
     for (i = 0; i < numBlocks; i++)
     {
@@ -125,8 +131,8 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize,
   size = (size_t)indexSize;
   if (size != indexSize)
     return SZ_ERROR_UNSUPPORTED;
-  buf = ISzAlloc_Alloc(alloc, size);
-  if (buf == 0)
+  buf = (Byte *)ISzAlloc_Alloc(alloc, size);
+  if (!buf)
     return SZ_ERROR_MEM;
   res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED);
   if (res == SZ_OK)
@@ -154,7 +160,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff
   pos -= XZ_STREAM_FOOTER_SIZE;
   RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE));
   
-  if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0)
+  if (!XZ_FOOTER_SIG_CHECK(buf + 10))
   {
     UInt32 total = 0;
     pos += XZ_STREAM_FOOTER_SIZE;
@@ -187,7 +193,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff
       return SZ_ERROR_NO_ARCHIVE;
     pos -= XZ_STREAM_FOOTER_SIZE;
     RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE));
-    if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0)
+    if (!XZ_FOOTER_SIG_CHECK(buf + 10))
       return SZ_ERROR_NO_ARCHIVE;
   }
   
@@ -262,7 +268,7 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p)
   UInt64 size = 0;
   size_t i;
   for (i = 0; i < p->num; i++)
-    ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i]));
+    ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i]));
   return size;
 }
 
@@ -272,7 +278,7 @@ UInt64 Xzs_GetPackSize(const CXzs *p)
   UInt64 size = 0;
   size_t i;
   for (i = 0; i < p->num; i++)
-    ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i]));
+    ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i]));
   return size;
 }
 */
@@ -294,7 +300,7 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr
     {
       size_t newNum = p->num + p->num / 4 + 1;
       Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream));
-      if (data == 0)
+      if (!data)
         return SZ_ERROR_MEM;
       p->numAllocated = newNum;
       if (p->num != 0)

CPP/7zip/Archive/7z/7zCompressionMode.h

@@ -13,7 +13,9 @@ struct CMethodFull: public CMethodProps
 {
   CMethodId Id;
   UInt32 NumStreams;
+  int CodecIndex;
 
+  CMethodFull(): CodecIndex(-1) {}
   bool IsSimpleCoder() const { return NumStreams == 1; }
 };
 

CPP/7zip/Archive/7z/7zDecode.cpp

@@ -236,8 +236,8 @@ HRESULT CDecoder::Decode(
     
     _7Z_DECODER_CRYPRO_VARS_DECL
 
-    #if !defined(_7ZIP_ST) && !defined(_SFX)
-    , bool mtMode, UInt32 numThreads
+    #if !defined(_7ZIP_ST)
+    , bool mtMode, UInt32 numThreads, UInt64 memUsage
     #endif
     )
 {
@@ -312,7 +312,7 @@ HRESULT CDecoder::Decode(
       #endif
   
       CCreatedCoder cod;
-      RINOK(CreateCoder(
+      RINOK(CreateCoder_Id(
           EXTERNAL_CODECS_LOC_VARS
           coderInfo.MethodID, false, cod));
     
@@ -355,11 +355,39 @@ HRESULT CDecoder::Decode(
 
   unsigned i;
 
+  bool mt_wasUsed = false;
+
   for (i = 0; i < folderInfo.Coders.Size(); i++)
   {
     const CCoderInfo &coderInfo = folderInfo.Coders[i];
     IUnknown *decoder = _mixer->GetCoder(i).GetUnknown();
 
+    #if !defined(_7ZIP_ST)
+    if (!mt_wasUsed)
+    {
+      if (mtMode)
+      {
+        CMyComPtr<ICompressSetCoderMt> setCoderMt;
+        decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt);
+        if (setCoderMt)
+        {
+          mt_wasUsed = true;
+          RINOK(setCoderMt->SetNumberOfThreads(numThreads));
+        }
+      }
+      // if (memUsage != 0)
+      {
+        CMyComPtr<ICompressSetMemLimit> setMemLimit;
+        decoder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit);
+        if (setMemLimit)
+        {
+          mt_wasUsed = true;
+          RINOK(setMemLimit->SetMemLimit(memUsage));
+        }
+      }
+    }
+    #endif
+
     {
       CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties;
       decoder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties);
@@ -376,18 +404,6 @@ HRESULT CDecoder::Decode(
       }
     }
 
-    #if !defined(_7ZIP_ST) && !defined(_SFX)
-    if (mtMode)
-    {
-      CMyComPtr<ICompressSetCoderMt> setCoderMt;
-      decoder->QueryInterface(IID_ICompressSetCoderMt, (void **)&setCoderMt);
-      if (setCoderMt)
-      {
-        RINOK(setCoderMt->SetNumberOfThreads(numThreads));
-      }
-    }
-    #endif
-
     #ifndef _NO_CRYPTO
     {
       CMyComPtr<ICryptoSetPassword> cryptoSetPassword;

CPP/7zip/Archive/7z/7zDecode.h

@@ -59,8 +59,8 @@ public:
       
       _7Z_DECODER_CRYPRO_VARS_DECL
       
-      #if !defined(_7ZIP_ST) && !defined(_SFX)
-      , bool mtMode, UInt32 numThreads
+      #if !defined(_7ZIP_ST)
+      , bool mtMode, UInt32 numThreads, UInt64 memUsage
       #endif
       );
 };

CPP/7zip/Archive/7z/7zEncode.cpp

@@ -154,9 +154,18 @@ HRESULT CEncoder::CreateMixerCoder(
 
     CCreatedCoder cod;
     
-    RINOK(CreateCoder(
+    if (methodFull.CodecIndex >= 0)
+    {
+      RINOK(CreateCoder_Index(
+        EXTERNAL_CODECS_LOC_VARS
+        methodFull.CodecIndex, true, cod));
+    }
+    else
+    {
+      RINOK(CreateCoder_Id(
         EXTERNAL_CODECS_LOC_VARS
         methodFull.Id, true, cod));
+    }
 
     if (cod.NumStreams != methodFull.NumStreams)
       return E_FAIL;
@@ -370,6 +379,17 @@ HRESULT CEncoder::Encode(
       resetInitVector->ResetInitVector();
     }
 
+    {
+      CMyComPtr<ICompressSetCoderPropertiesOpt> optProps;
+      coder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps);
+      if (optProps)
+      {
+        PROPID propID = NCoderPropID::kExpectedDataSize;
+        NWindows::NCOM::CPropVariant prop = (UInt64)unpackSize;
+        RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1));
+      }
+    }
+
     CMyComPtr<ICompressWriteCoderProperties> writeCoderProperties;
     coder->QueryInterface(IID_ICompressWriteCoderProperties, (void **)&writeCoderProperties);
 
@@ -380,7 +400,7 @@ HRESULT CEncoder::Encode(
       CDynBufSeqOutStream *outStreamSpec = new CDynBufSeqOutStream;
       CMyComPtr<ISequentialOutStream> dynOutStream(outStreamSpec);
       outStreamSpec->Init();
-      writeCoderProperties->WriteCoderProperties(dynOutStream);
+      RINOK(writeCoderProperties->WriteCoderProperties(dynOutStream));
       outStreamSpec->CopyToBuffer(props);
     }
     else

CPP/7zip/Archive/7z/7zExtract.cpp

@@ -152,6 +152,12 @@ STDMETHODIMP CFolderOutStream::Write(const void *data, UInt32 size, UInt32 *proc
     if (_fileIsOpen)
     {
       UInt32 cur = (size < _rem ? size : (UInt32)_rem);
+      if (_calcCrc)
+      {
+        const UInt32 k_Step = (UInt32)1 << 20;
+        if (cur > k_Step)
+          cur = k_Step;
+      }
       HRESULT result = S_OK;
       if (_stream)
         result = _stream->Write(data, cur, &cur);
@@ -363,8 +369,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
           , dataAfterEnd_Error
           
           _7Z_DECODER_CRYPRO_VARS
-          #if !defined(_7ZIP_ST) && !defined(_SFX)
-            , true, _numThreads
+          #if !defined(_7ZIP_ST)
+            , true, _numThreads, _memUsage
           #endif
           );
 

CPP/7zip/Archive/7z/7zFolderInStream.cpp

@@ -80,14 +80,17 @@ STDMETHODIMP CFolderInStream::Read(void *data, UInt32 size, UInt32 *processedSiz
   {
     if (_stream)
     {
-      UInt32 processed2;
-      RINOK(_stream->Read(data, size, &processed2));
-      if (processed2 != 0)
+      UInt32 cur = size;
+      const UInt32 kMax = (UInt32)1 << 20;
+      if (cur > kMax)
+        cur = kMax;
+      RINOK(_stream->Read(data, cur, &cur));
+      if (cur != 0)
       {
-        _crc = CrcUpdate(_crc, data, processed2);
-        _pos += processed2;
+        _crc = CrcUpdate(_crc, data, cur);
+        _pos += cur;
         if (processedSize)
-          *processedSize = processed2;
+          *processedSize = cur;
         return S_OK;
       }
       

CPP/7zip/Archive/7z/7zHandler.cpp

@@ -40,7 +40,6 @@ CHandler::CHandler()
   _crcSize = 4;
   
   #ifdef __7Z_SET_PROPERTIES
-  _numThreads = NSystem::GetNumberOfProcessors();
   _useMultiThreadMixer = true;
   #endif
   
@@ -714,8 +713,8 @@ STDMETHODIMP CHandler::Close()
 STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
 {
   COM_TRY_BEGIN
-  const UInt32 numProcessors = NSystem::GetNumberOfProcessors();
-  _numThreads = numProcessors;
+  
+  InitCommon();
   _useMultiThreadMixer = true;
 
   for (UInt32 i = 0; i < numProps; i++)
@@ -734,13 +733,15 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR
         RINOK(PROPVARIANT_to_bool(value, _useMultiThreadMixer));
         continue;
       }
-      if (name.IsPrefixedBy_Ascii_NoCase("mt"))
       {
-        RINOK(ParseMtProp(name.Ptr(2), value, numProcessors, _numThreads));
-        continue;
+        HRESULT hres;
+        if (SetCommonProperty(name, value, hres))
+        {
+          RINOK(hres);
+          continue;
+        }
       }
-      else
-        return E_INVALIDARG;
+      return E_INVALIDARG;
     }
   }
   return S_OK;

CPP/7zip/Archive/7z/7zHandler.h

@@ -8,16 +8,6 @@
 
 #include "../../Common/CreateCoder.h"
 
-#ifndef EXTRACT_ONLY
-#include "../Common/HandlerOut.h"
-#endif
-
-#include "7zCompressionMode.h"
-#include "7zIn.h"
-
-namespace NArchive {
-namespace N7z {
-
 #ifndef __7Z_SET_PROPERTIES
 
 #ifdef EXTRACT_ONLY
@@ -30,6 +20,16 @@ namespace N7z {
 
 #endif
 
+// #ifdef __7Z_SET_PROPERTIES
+#include "../Common/HandlerOut.h"
+// #endif
+
+#include "7zCompressionMode.h"
+#include "7zIn.h"
+
+namespace NArchive {
+namespace N7z {
+
 
 #ifndef EXTRACT_ONLY
 
@@ -38,8 +38,6 @@ class COutHandler: public CMultiMethodProps
   HRESULT SetSolidFromString(const UString &s);
   HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value);
 public:
-  bool _removeSfxBlock;
-  
   UInt64 _numSolidFiles;
   UInt64 _numSolidBytes;
   bool _numSolidBytesDefined;
@@ -58,6 +56,8 @@ public:
 
   bool _useMultiThreadMixer;
 
+  bool _removeSfxBlock;
+  
   // bool _volumeMode;
 
   void InitSolidFiles() { _numSolidFiles = (UInt64)(Int64)(-1); }
@@ -70,9 +70,10 @@ public:
     _numSolidBytesDefined = false;
   }
 
+  void InitProps7z();
   void InitProps();
 
-  COutHandler() { InitProps(); }
+  COutHandler() { InitProps7z(); }
 
   HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value);
 };
@@ -82,16 +83,23 @@ public:
 class CHandler:
   public IInArchive,
   public IArchiveGetRawProps,
+  
   #ifdef __7Z_SET_PROPERTIES
   public ISetProperties,
   #endif
+  
   #ifndef EXTRACT_ONLY
   public IOutArchive,
   #endif
+  
   PUBLIC_ISetCompressCodecsInfo
-  public CMyUnknownImp
+  
+  public CMyUnknownImp,
+
   #ifndef EXTRACT_ONLY
-  , public COutHandler
+    public COutHandler
+  #else
+    public CCommonMethodProps
   #endif
 {
 public:
@@ -135,7 +143,6 @@ private:
   #ifdef EXTRACT_ONLY
   
   #ifdef __7Z_SET_PROPERTIES
-  UInt32 _numThreads;
   bool _useMultiThreadMixer;
   #endif
 

CPP/7zip/Archive/7z/7zHandlerOut.cpp

@@ -13,6 +13,8 @@
 #include "7zOut.h"
 #include "7zUpdate.h"
 
+#ifndef EXTRACT_ONLY
+
 using namespace NWindows;
 
 namespace NArchive {
@@ -41,9 +43,11 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *type)
 
 HRESULT CHandler::PropsMethod_To_FullMethod(CMethodFull &dest, const COneMethodInfo &m)
 {
-  if (!FindMethod(
+  dest.CodecIndex = FindMethod_Index(
       EXTERNAL_CODECS_VARS
-      m.MethodName, dest.Id, dest.NumStreams))
+      m.MethodName, true,
+      dest.Id, dest.NumStreams);
+  if (dest.CodecIndex < 0)
     return E_INVALIDARG;
   (CProps &)dest = (CProps &)m;
   return S_OK;
@@ -699,10 +703,8 @@ static HRESULT ParseBond(UString &srcString, UInt32 &coder, UInt32 &stream)
   return S_OK;
 }
 
-void COutHandler::InitProps()
+void COutHandler::InitProps7z()
 {
-  CMultiMethodProps::Init();
-
   _removeSfxBlock = false;
   _compressHeaders = true;
   _encryptHeadersSpecified = false;
@@ -722,6 +724,14 @@ void COutHandler::InitProps()
   _useTypeSorting = false;
 }
 
+void COutHandler::InitProps()
+{
+  CMultiMethodProps::Init();
+  InitProps7z();
+}
+
+
+
 HRESULT COutHandler::SetSolidFromString(const UString &s)
 {
   UString s2 = s;
@@ -762,6 +772,10 @@ HRESULT COutHandler::SetSolidFromString(const UString &s)
       }
       _numSolidBytes = (v << numBits);
       _numSolidBytesDefined = true;
+      /*
+      if (_numSolidBytes == 0)
+        _numSolidFiles = 1;
+      */
     }
   }
   return S_OK;
@@ -810,7 +824,7 @@ HRESULT COutHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &val
       return E_INVALIDARG;
     return SetSolidFromString(name);
   }
-  
+
   UInt32 number;
   int index = ParseStringToUInt32(name, number);
   // UString realName = name.Ptr(index);
@@ -921,3 +935,5 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR
 }
 
 }}
+
+#endif

CPP/7zip/Archive/7z/7zIn.cpp

@@ -1115,11 +1115,11 @@ HRESULT CInArchive::ReadAndDecodePackedStreams(
         , dataAfterEnd_Error
         
         _7Z_DECODER_CRYPRO_VARS
-        #if !defined(_7ZIP_ST) && !defined(_SFX)
+        #if !defined(_7ZIP_ST)
           , false // mtMode
           , 1     // numThreads
+          , 0     // memUsage
         #endif
-
       );
     
     RINOK(result);

CPP/7zip/Archive/7z/7zOut.cpp

@@ -515,7 +515,7 @@ HRESULT COutArchive::EncodeStream(
   outFolders.FolderUnpackCRCs.Vals.Add(CrcCalc(data, data.Size()));
   // outFolders.NumUnpackStreamsVector.Add(1);
   UInt64 dataSize64 = data.Size();
-  UInt64 unpackSize;
+  UInt64 unpackSize = data.Size();
   RINOK(encoder.Encode(
       EXTERNAL_CODECS_LOC_VARS
       stream,

CPP/7zip/Archive/7z/7zUpdate.cpp

@@ -1507,7 +1507,8 @@ void CThreadDecoder::Execute()
 
       _7Z_DECODER_CRYPRO_VARS
       #ifndef _7ZIP_ST
-        , MtMode, NumThreads
+        , MtMode, NumThreads,
+        0 // MemUsage
       #endif
 
       );
@@ -1696,13 +1697,14 @@ HRESULT Update(
 
   UInt64 inSizeForReduce = 0;
   {
+    bool isSolid = (numSolidFiles > 1 && options.NumSolidBytes != 0);
     FOR_VECTOR (i, updateItems)
     {
       const CUpdateItem &ui = updateItems[i];
       if (ui.NewData)
       {
         complexity += ui.Size;
-        if (numSolidFiles != 1)
+        if (isSolid)
           inSizeForReduce += ui.Size;
         else if (inSizeForReduce < ui.Size)
           inSizeForReduce = ui.Size;
@@ -2142,8 +2144,8 @@ HRESULT Update(
                   #ifndef _7ZIP_ST
                     , false // mtMode
                     , 1 // numThreads
+                    , 0 // memUsage
                   #endif
-
                 );
           
               RINOK(res);
@@ -2171,6 +2173,7 @@ HRESULT Update(
             #endif
           }
 
+          curUnpackSize = sizeToEncode;
 
           HRESULT encodeRes = encoder.Encode(
               EXTERNAL_CODECS_LOC_VARS
@@ -2292,7 +2295,8 @@ HRESULT Update(
       continue;
     CRecordVector<CRefItem> refItems;
     refItems.ClearAndSetSize(numFiles);
-    bool sortByType = (options.UseTypeSorting && numSolidFiles > 1);
+    // bool sortByType = (options.UseTypeSorting && isSoid); // numSolidFiles > 1
+    bool sortByType = options.UseTypeSorting;
     
     unsigned i;
 
@@ -2358,7 +2362,9 @@ HRESULT Update(
       inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
       
       unsigned startPackIndex = newDatabase.PackSizes.Size();
-      UInt64 curFolderUnpackSize;
+      UInt64 curFolderUnpackSize = totalSize;
+      // curFolderUnpackSize = (UInt64)(Int64)-1;
+      
       RINOK(encoder.Encode(
           EXTERNAL_CODECS_LOC_VARS
           solidInStream,

CPP/7zip/Archive/Common/HandlerOut.cpp

@@ -2,18 +2,92 @@
 
 #include "StdAfx.h"
 
-#ifndef _7ZIP_ST
-#include "../../../Windows/System.h"
-#endif
+#include "../../../Common/StringToInt.h"
 
 #include "../Common/ParseProperties.h"
 
 #include "HandlerOut.h"
 
-using namespace NWindows;
-
 namespace NArchive {
 
+bool ParseSizeString(const wchar_t *s, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res)
+{
+  if (*s == 0)
+  {
+    switch (prop.vt)
+    {
+      case VT_UI4: res = prop.ulVal; return true;
+      case VT_UI8: res = prop.uhVal.QuadPart; return true;
+      case VT_BSTR:
+        s = prop.bstrVal;
+        break;
+      default: return false;
+    }
+  }
+  else if (prop.vt != VT_EMPTY)
+    return false;
+
+  const wchar_t *end;
+  UInt64 v = ConvertStringToUInt64(s, &end);
+  if (s == end)
+    return false;
+  wchar_t c = *end;
+  if (c == 0)
+  {
+    res = v;
+    return true;
+  }
+  if (end[1] != 0)
+    return false;
+
+  if (c == '%')
+  {
+    res = percentsBase / 100 * v;
+    return true;
+  }
+
+  unsigned numBits;
+  switch (MyCharLower_Ascii(c))
+  {
+    case 'b': numBits =  0; break;
+    case 'k': numBits = 10; break;
+    case 'm': numBits = 20; break;
+    case 'g': numBits = 30; break;
+    case 't': numBits = 40; break;
+    default: return false;
+  }
+  UInt64 val2 = v << numBits;
+  if ((val2 >> numBits) != v)
+    return false;
+  res = val2;
+  return true;
+}
+
+bool CCommonMethodProps::SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres)
+{
+  hres = S_OK;
+
+  if (name.IsPrefixedBy_Ascii_NoCase("mt"))
+  {
+    #ifndef _7ZIP_ST
+    hres = ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads);
+    #endif
+    return true;
+  }
+  
+  if (name.IsPrefixedBy_Ascii_NoCase("memuse"))
+  {
+    if (!ParseSizeString(name.Ptr(6), value, _memAvail, _memUsage))
+      hres = E_INVALIDARG;
+    return true;
+  }
+
+  return false;
+}
+
+
+#ifndef EXTRACT_ONLY
+
 static void SetMethodProp32(COneMethodInfo &m, PROPID propID, UInt32 value)
 {
   if (m.FindProp(propID) < 0)
@@ -34,21 +108,23 @@ void CMultiMethodProps::SetMethodThreadsTo(COneMethodInfo &oneMethodInfo, UInt32
 }
 #endif
 
-void CMultiMethodProps::Init()
+void CMultiMethodProps::InitMulti()
 {
-  #ifndef _7ZIP_ST
-  _numProcessors = _numThreads = NSystem::GetNumberOfProcessors();
-  #endif
-  
   _level = (UInt32)(Int32)-1;
   _analysisLevel = -1;
-
-  _autoFilter = true;
   _crcSize = 4;
-  _filterMethod.Clear();
-  _methods.Clear();
+  _autoFilter = true;
 }
 
+void CMultiMethodProps::Init()
+{
+  InitCommon();
+  InitMulti();
+  _methods.Clear();
+  _filterMethod.Clear();
+}
+
+
 HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value)
 {
   UString name = nameSpec;
@@ -78,20 +154,18 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN
     _crcSize = 4;
     return ParsePropToUInt32(name, value, _crcSize);
   }
+
+  {
+    HRESULT hres;
+    if (SetCommonProperty(name, value, hres))
+      return hres;
+  }
   
   UInt32 number;
   unsigned index = ParseStringToUInt32(name, number);
   UString realName = name.Ptr(index);
   if (index == 0)
   {
-    if (name.IsPrefixedBy_Ascii_NoCase("mt"))
-    {
-      #ifndef _7ZIP_ST
-      RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads));
-      #endif
-      
-      return S_OK;
-    }
     if (name.IsEqualTo("f"))
     {
       HRESULT res = PROPVARIANT_to_bool(value, _autoFilter);
@@ -110,20 +184,20 @@ HRESULT CMultiMethodProps::SetProperty(const wchar_t *nameSpec, const PROPVARIAN
   return _methods[number].ParseMethodFromPROPVARIANT(realName, value);
 }
 
+
+
 void CSingleMethodProps::Init()
 {
+  InitCommon();
+  InitSingle();
   Clear();
-  _level = (UInt32)(Int32)-1;
-  
-  #ifndef _7ZIP_ST
-  _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors();
-  AddProp_NumThreads(_numThreads);
-  #endif
 }
 
+
 HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
 {
   Init();
+  
   for (UInt32 i = 0; i < numProps; i++)
   {
     UString name = names[i];
@@ -137,20 +211,22 @@ HRESULT CSingleMethodProps::SetProperties(const wchar_t * const *names, const PR
       RINOK(ParsePropToUInt32(name.Ptr(1), value, a));
       _level = a;
       AddProp_Level(a);
+      continue;
     }
-    else if (name.IsPrefixedBy_Ascii_NoCase("mt"))
     {
-      #ifndef _7ZIP_ST
-      RINOK(ParseMtProp(name.Ptr(2), value, _numProcessors, _numThreads));
-      AddProp_NumThreads(_numThreads);
-      #endif
-    }
-    else
-    {
-      RINOK(ParseMethodFromPROPVARIANT(names[i], value));
+      HRESULT hres;
+      if (SetCommonProperty(name, value, hres))
+      {
+        RINOK(hres)
+        continue;
+      }
     }
+    RINOK(ParseMethodFromPROPVARIANT(names[i], value));
   }
+
   return S_OK;
 }
 
+#endif
+
 }

CPP/7zip/Archive/Common/HandlerOut.h

@@ -3,20 +3,57 @@
 #ifndef __HANDLER_OUT_H
 #define __HANDLER_OUT_H
 
+#include "../../../Windows/System.h"
+
 #include "../../Common/MethodProps.h"
 
 namespace NArchive {
 
-class CMultiMethodProps
+bool ParseSizeString(const wchar_t *name, const PROPVARIANT &prop, UInt64 percentsBase, UInt64 &res);
+
+class CCommonMethodProps
 {
-  UInt32 _level;
-  int _analysisLevel;
+protected:
+  void InitCommon()
+  {
+    #ifndef _7ZIP_ST
+    _numProcessors = _numThreads = NWindows::NSystem::GetNumberOfProcessors();
+    #endif
+
+    UInt64 memAvail = (UInt64)(sizeof(size_t)) << 28;
+    _memAvail = memAvail;
+    _memUsage = memAvail;
+    if (NWindows::NSystem::GetRamSize(memAvail))
+    {
+      _memAvail = memAvail;
+      _memUsage = memAvail / 32 * 17;
+    }
+  }
+
 public:
   #ifndef _7ZIP_ST
   UInt32 _numThreads;
   UInt32 _numProcessors;
   #endif
 
+  UInt64 _memUsage;
+  UInt64 _memAvail;
+
+  bool SetCommonProperty(const UString &name, const PROPVARIANT &value, HRESULT &hres);
+
+  CCommonMethodProps() { InitCommon(); }
+};
+
+
+#ifndef EXTRACT_ONLY
+
+class CMultiMethodProps: public CCommonMethodProps
+{
+  UInt32 _level;
+  int _analysisLevel;
+
+  void InitMulti();
+public:
   UInt32 _crcSize;
   CObjectVector<COneMethodInfo> _methods;
   COneMethodInfo _filterMethod;
@@ -43,27 +80,31 @@ public:
   int GetAnalysisLevel() const { return _analysisLevel; }
 
   void Init();
+  CMultiMethodProps() { InitMulti(); }
 
-  CMultiMethodProps() { Init(); }
   HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value);
 };
 
-class CSingleMethodProps: public COneMethodInfo
+
+class CSingleMethodProps: public COneMethodInfo, public CCommonMethodProps
 {
   UInt32 _level;
-  
-public:
-  #ifndef _7ZIP_ST
-  UInt32 _numThreads;
-  UInt32 _numProcessors;
-  #endif
 
+  void InitSingle()
+  {
+    _level = (UInt32)(Int32)-1;
+  }
+
+public:
   void Init();
-  CSingleMethodProps() { Init(); }
+  CSingleMethodProps() { InitSingle(); }
+  
   int GetLevel() const { return _level == (UInt32)(Int32)-1 ? 5 : (int)_level; }
   HRESULT SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
 };
 
+#endif
+
 }
 
 #endif

CPP/7zip/Archive/DmgHandler.cpp

@@ -19,6 +19,7 @@
 
 #include "../Compress/BZip2Decoder.h"
 #include "../Compress/CopyCoder.h"
+#include "../Compress/LzfseDecoder.h"
 #include "../Compress/ZlibDecoder.h"
 
 #include "Common/OutStreamWithCRC.h"
@@ -121,6 +122,7 @@ enum
   METHOD_ADC    = 0x80000004,
   METHOD_ZLIB   = 0x80000005,
   METHOD_BZIP2  = 0x80000006,
+  METHOD_LZFSE  = 0x80000007,
   METHOD_COMMENT = 0x7FFFFFFE, // is used to comment "+beg" and "+end" in extra field.
   METHOD_END    = 0xFFFFFFFF
 };
@@ -276,6 +278,7 @@ void CMethods::GetString(AString &res) const
       case METHOD_ADC:    s = "ADC";   break;
       case METHOD_ZLIB:   s = "ZLIB";  break;
       case METHOD_BZIP2:  s = "BZip2"; break;
+      case METHOD_LZFSE:  s = "LZFSE"; break;
       default: ConvertUInt32ToString(type, buf); s = buf;
     }
     res.Add_OptSpaced(s);
@@ -307,6 +310,10 @@ static const CAppleName k_Names[] =
   { true,  "hfs",  "Apple_HFS" },
   { true,  "hfsx", "Apple_HFSX" },
   { true,  "ufs",  "Apple_UFS" },
+
+  // efi_sys partition is FAT32, but it's not main file. So we use (IsFs = false)
+  { false,  "efi_sys", "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" },
+
   { false, "free", "Apple_Free" },
   { false, "ddm",  "DDM" },
   { false, NULL,   "Apple_partition_map" },
@@ -1247,6 +1254,11 @@ STDMETHODIMP CAdcDecoder::Code(ISequentialInStream *inStream,
 }
 
 
+
+
+
+
+
 STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
     Int32 testMode, IArchiveExtractCallback *extractCallback)
 {
@@ -1292,6 +1304,9 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
   CAdcDecoder *adcCoderSpec = new CAdcDecoder();
   CMyComPtr<ICompressCoder> adcCoder = adcCoderSpec;
 
+  NCompress::NLzfse::CDecoder *lzfseCoderSpec = new NCompress::NLzfse::CDecoder();
+  CMyComPtr<ICompressCoder> lzfseCoder = lzfseCoderSpec;
+
   CLocalProgress *lps = new CLocalProgress;
   CMyComPtr<ICompressProgressInfo> progress = lps;
   lps->Init(extractCallback, false);
@@ -1419,6 +1434,12 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
                   opRes = NExtract::NOperationResult::kDataError;
               break;
             }
+
+            case METHOD_LZFSE:
+            {
+              res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, progress);
+              break;
+            }
             
             default:
               opRes = NExtract::NOperationResult::kUnsupportedMethod;
@@ -1490,6 +1511,9 @@ class CInStream:
   CAdcDecoder *adcCoderSpec;
   CMyComPtr<ICompressCoder> adcCoder;
 
+  NCompress::NLzfse::CDecoder *lzfseCoderSpec;
+  CMyComPtr<ICompressCoder> lzfseCoder;
+
   CBufPtrSeqOutStream *outStreamSpec;
   CMyComPtr<ISequentialOutStream> outStream;
 
@@ -1651,6 +1675,15 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
             if (res == S_OK && bzip2CoderSpec->GetInputProcessedSize() != block.PackSize)
               res = S_FALSE;
             break;
+
+          case METHOD_LZFSE:
+            if (!lzfseCoder)
+            {
+              lzfseCoderSpec = new NCompress::NLzfse::CDecoder();
+              lzfseCoder = lzfseCoderSpec;
+            }
+            res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, NULL);
+            break;
             
           default:
             return E_FAIL;
@@ -1738,6 +1771,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream)
       case METHOD_ADC:
       case METHOD_ZLIB:
       case METHOD_BZIP2:
+      case METHOD_LZFSE:
       case METHOD_END:
         break;
       default:

CPP/7zip/Archive/ExtHandler.cpp

@@ -461,8 +461,8 @@ bool CHeader::Parse(const Byte *p)
 
   if (Is64Bit())
   {
-    HI_32(150, NumBlocks);
-    // HI_32(154, NumBlocksSuper);
+    HI_32(0x150, NumBlocks);
+    // HI_32(0x154, NumBlocksSuper);
     HI_32(0x158, NumFreeBlocks);
   }
 

CPP/7zip/Archive/IArchive.h

@@ -488,6 +488,16 @@ ARCHIVE_INTERFACE(IOutArchive, 0xA0)
 };
 
 
+/*
+ISetProperties::SetProperties()
+  PROPVARIANT values[i].vt:
+    VT_EMPTY
+    VT_BOOL
+    VT_UI4   - if 32-bit number
+    VT_UI8   - if 64-bit number
+    VT_BSTR
+*/
+
 ARCHIVE_INTERFACE(ISetProperties, 0x03)
 {
   STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps) PURE;

CPP/7zip/Archive/Iso/IsoIn.cpp

@@ -180,7 +180,7 @@ UInt32 CInArchive::ReadDigits(int numDigits)
     Byte b = ReadByte();
     if (b < '0' || b > '9')
     {
-      if (b == 0) // it's bug in some CD's
+      if (b == 0 || b == ' ') // it's bug in some CD's
         b = '0';
       else
         throw CHeaderErrorException();

CPP/7zip/Archive/Nsis/NsisIn.cpp

@@ -2489,7 +2489,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p)
           if (c2 == NS_3_CODE_VAR)
             // if (c2 <= NS_3_CODE_SKIP && c2 != NS_3_CODE_SHELL && c2 != 1)
           {
-            if ((strData[i+ 2] & 0x80) != 0)
+            if ((strData[i + 2] & 0x80) != 0)
             {
               // const char *p2 = (const char *)(strData + i + 1);
               // p2 = p2;
@@ -5048,6 +5048,12 @@ HRESULT CInArchive::Parse()
   DetectNsisType(bhEntries, _data + bhEntries.Offset);
 
   Decoder.IsNsisDeflate = (NsisType != k_NsisType_Nsis3);
+  
+  // some NSIS files (that are not detected as k_NsisType_Nsis3)
+  // use original (non-NSIS) Deflate
+  // How to detect these cases?
+
+  // Decoder.IsNsisDeflate = false;
 
 
   #ifdef NSIS_SCRIPT

CPP/7zip/Archive/NtfsHandler.cpp

@@ -341,15 +341,19 @@ bool CVolInfo::Parse(const Byte *p, unsigned size)
 struct CAttr
 {
   UInt32 Type;
+
+  Byte NonResident;
+
+  // Non-Resident
+  Byte CompressionUnit;
+
   // UInt32 Len;
   UString2 Name;
   // UInt16 Flags;
   // UInt16 Instance;
   CByteBuffer Data;
-  Byte NonResident;
 
   // Non-Resident
-  Byte CompressionUnit;
   UInt64 LowVcn;
   UInt64 HighVcn;
   UInt64 AllocatedSize;
@@ -408,15 +412,16 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
     return 8; // required size is 4, but attributes are 8 bytes aligned. So we return 8
   if (size < 0x18)
     return 0;
+
   PRF(printf(" T=%2X", Type));
   
-  UInt32 len = Get32(p + 0x04);
+  UInt32 len = Get32(p + 4);
   PRF(printf(" L=%3d", len));
   if (len > size)
     return 0;
   if ((len & 7) != 0)
     return 0;
-  NonResident = p[0x08];
+  NonResident = p[8];
   {
     unsigned nameLen = p[9];
     UInt32 nameOffset = Get16(p + 0x0A);
@@ -437,6 +442,7 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
 
   UInt32 dataSize;
   UInt32 offs;
+  
   if (NonResident)
   {
     if (len < 0x40)
@@ -472,16 +478,19 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
   {
     if (len < 0x18)
       return 0;
-    PRF(printf(" RES"));
-    dataSize = Get32(p + 0x10);
-    PRF(printf(" dataSize=%3d", dataSize));
-    offs = Get16(p + 0x14);
+    G32(p + 0x10, dataSize);
+    G16(p + 0x14, offs);
     // G16(p + 0x16, ResidentFlags);
+    PRF(printf(" RES"));
+    PRF(printf(" dataSize=%3d", dataSize));
     // PRF(printf(" ResFlags=%4X", ResidentFlags));
   }
+  
   if (offs > len || dataSize > len || len - dataSize < offs)
     return 0;
+  
   Data.CopyFrom(p + offs, dataSize);
+  
   #ifdef SHOW_DEBUG_INFO
   PRF(printf("  : "));
   for (unsigned i = 0; i < Data.Size(); i++)
@@ -489,16 +498,19 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
     PRF(printf(" %02X", (unsigned)Data[i]));
   }
   #endif
+  
   return len;
 }
 
+
 bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax, unsigned compressionUnit) const
 {
   const Byte *p = Data;
   unsigned size = (unsigned)Data.Size();
   UInt64 vcn = LowVcn;
   UInt64 lcn = 0;
-  UInt64 highVcn1 = HighVcn + 1;
+  const UInt64 highVcn1 = HighVcn + 1;
+  
   if (LowVcn != extents.Back().Virt || highVcn1 > (UInt64)1 << 63)
     return false;
 
@@ -528,15 +540,30 @@ bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax,
     if ((highVcn1 - vcn) < vSize)
       return false;
 
+    CExtent e;
+    e.Virt = vcn;
+    vcn += vSize;
+
     num = (b >> 4) & 0xF;
     if (num > 8 || num > size)
       return false;
-    CExtent e;
-    e.Virt = vcn;
+    
     if (num == 0)
     {
+      // Sparse
+      
+      /* if Unit is compressed, it can have many Elements for each compressed Unit:
+         and last Element for unit MUST be without LCN.
+           Element 0: numCompressedClusters2, LCN_0
+           Element 1: numCompressedClusters2, LCN_1
+           ...
+           Last Element : (16 - total_clusters_in_previous_elements), no LCN
+      */
+      
+      // sparse is not allowed for (compressionUnit == 0) ? Why ?
       if (compressionUnit == 0)
         return false;
+
       e.Phy = kEmptyExtent;
     }
     else
@@ -553,9 +580,10 @@ bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax,
         return false;
       e.Phy = lcn;
     }
+    
     extents.Add(e);
-    vcn += vSize;
   }
+
   CExtent e;
   e.Phy = kEmptyExtent;
   e.Virt = vcn;
@@ -563,10 +591,11 @@ bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax,
   return (highVcn1 == vcn);
 }
 
+
 static const UInt64 kEmptyTag = (UInt64)(Int64)-1;
 
 static const unsigned kNumCacheChunksLog = 1;
-static const size_t kNumCacheChunks = (1 << kNumCacheChunksLog);
+static const size_t kNumCacheChunks = (size_t)1 << kNumCacheChunksLog;
 
 class CInStream:
   public IInStream,
@@ -734,24 +763,27 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
 
   while (_curRem == 0)
   {
-    UInt64 cacheTag = _virtPos >> _chunkSizeLog;
-    UInt32 cacheIndex = (UInt32)cacheTag & (kNumCacheChunks - 1);
+    const UInt64 cacheTag = _virtPos >> _chunkSizeLog;
+    const size_t cacheIndex = (size_t)cacheTag & (kNumCacheChunks - 1);
+    
     if (_tags[cacheIndex] == cacheTag)
     {
-      UInt32 chunkSize = (UInt32)1 << _chunkSizeLog;
-      UInt32 offset = (UInt32)_virtPos & (chunkSize - 1);
-      UInt32 cur = MyMin(chunkSize - offset, size);
+      const size_t chunkSize = (size_t)1 << _chunkSizeLog;
+      const size_t offset = (size_t)_virtPos & (chunkSize - 1);
+      size_t cur = chunkSize - offset;
+      if (cur > size)
+        cur = size;
       memcpy(data, _outBuf + (cacheIndex << _chunkSizeLog) + offset, cur);
-      *processedSize = cur;
+      *processedSize = (UInt32)cur;
       _virtPos += cur;
       return S_OK;
     }
 
     PRF2(printf("\nVirtPos = %6d", _virtPos));
     
-    UInt32 comprUnitSize = (UInt32)1 << CompressionUnit;
-    UInt64 virtBlock = _virtPos >> BlockSizeLog;
-    UInt64 virtBlock2 = virtBlock & ~((UInt64)comprUnitSize - 1);
+    const UInt32 comprUnitSize = (UInt32)1 << CompressionUnit;
+    const UInt64 virtBlock = _virtPos >> BlockSizeLog;
+    const UInt64 virtBlock2 = virtBlock & ~((UInt64)comprUnitSize - 1);
     
     unsigned left = 0, right = Extents.Size();
     for (;;)
@@ -766,7 +798,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
     }
     
     bool isCompressed = false;
-    UInt64 virtBlock2End = virtBlock2 + comprUnitSize;
+    const UInt64 virtBlock2End = virtBlock2 + comprUnitSize;
     if (CompressionUnit != 0)
       for (unsigned i = left; i < Extents.Size(); i++)
       {
@@ -802,7 +834,9 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
       _curRem = next - _virtPos;
       break;
     }
+    
     bool thereArePhy = false;
+    
     for (unsigned i2 = left; i2 < Extents.Size(); i2++)
     {
       const CExtent &e = Extents[i2];
@@ -814,6 +848,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
         break;
       }
     }
+    
     if (!thereArePhy)
     {
       _curRem = (Extents[i + 1].Virt << BlockSizeLog) - _virtPos;
@@ -823,6 +858,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
     
     size_t offs = 0;
     UInt64 curVirt = virtBlock2;
+    
     for (i = left; i < Extents.Size(); i++)
     {
       const CExtent &e = Extents[i];
@@ -845,6 +881,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
       _physPos += compressed;
       offs += compressed;
     }
+    
     size_t destLenMax = GetCuSize();
     size_t destLen = destLenMax;
     const UInt64 rem = Size - (virtBlock2 << BlockSizeLog);
@@ -863,6 +900,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
         return S_FALSE;
     }
   }
+  
   if (size > _curRem)
     size = (UInt32)_curRem;
   HRESULT res = S_OK;
@@ -913,6 +951,12 @@ static HRESULT DataParseExtents(unsigned clusterSizeLog, const CObjectVector<CAt
   
   const CAttr &attr0 = attrs[attrIndex];
 
+  /*
+  if (attrs[attrIndexLim - 1].HighVcn + 1 != (attr0.AllocatedSize >> clusterSizeLog))
+  {
+  }
+  */
+
   if (attr0.AllocatedSize < attr0.Size ||
       (attrs[attrIndexLim - 1].HighVcn + 1) != (attr0.AllocatedSize >> clusterSizeLog) ||
       (attr0.AllocatedSize & ((1 << clusterSizeLog) - 1)) != 0)

CPP/7zip/Archive/PeHandler.cpp

@@ -109,7 +109,8 @@ void CVersion::ToProp(NCOM::CPropVariant &prop)
   prop = sz;
 }
 
-static const unsigned kHeaderSize = 4 + 20;
+static const unsigned kCoffHeaderSize = 20;
+static const unsigned kPeHeaderSize = 4 + kCoffHeaderSize;
 static const unsigned k_OptHeader32_Size_MIN = 96;
 static const unsigned k_OptHeader64_Size_MIN = 112;
 
@@ -125,15 +126,14 @@ struct CHeader
   UInt16 OptHeaderSize;
   UInt16 Flags;
 
-  bool Parse(const Byte *p);
+  void ParseBase(const Byte *p);
+  bool ParseCoff(const Byte *p);
+  bool ParsePe(const Byte *p);
   bool IsDll() const { return (Flags & PE_IMAGE_FILE_DLL) != 0; }
 };
 
-bool CHeader::Parse(const Byte *p)
+void CHeader::ParseBase(const Byte *p)
 {
-  if (Get32(p) != k_Signature32)
-    return false;
-  p += 4;
   G16( 0, Machine);
   G16( 2, NumSections);
   G32( 4, Time);
@@ -141,6 +141,13 @@ bool CHeader::Parse(const Byte *p)
   G32(12, NumSymbols);
   G16(16, OptHeaderSize);
   G16(18, Flags);
+}
+
+bool CHeader::ParsePe(const Byte *p)
+{
+  if (Get32(p) != k_Signature32)
+    return false;
+  ParseBase(p + 4);
   return OptHeaderSize >= k_OptHeader32_Size_MIN;
 }
 
@@ -380,6 +387,10 @@ void CSection::Parse(const Byte *p)
   G32(36, Flags);
 }
 
+
+
+// IMAGE_FILE_*
+
 static const CUInt32PCharPair g_HeaderCharacts[] =
 {
   {  1, "Executable" },
@@ -399,39 +410,65 @@ static const CUInt32PCharPair g_HeaderCharacts[] =
   { 15, "Big-Endian" }
 };
 
-static const CUInt32PCharPair g_DllCharacts[] =
+// IMAGE_DLLCHARACTERISTICS_*
+
+static const char * const g_DllCharacts[] =
 {
-  {  5, "HighEntropyVA" },
-  {  6, "Relocated" },
-  {  7, "Integrity" },
-  {  8, "NX-Compatible" },
-  {  9, "NoIsolation" },
-  { 10, "NoSEH" },
-  { 11, "NoBind" },
-  { 12, "AppContainer" },
-  { 13, "WDM" },
-  { 14, "GuardCF" },
-  { 15, "TerminalServerAware" }
+    NULL
+  , NULL
+  , NULL
+  , NULL
+  , NULL
+  , "HighEntropyVA"
+  , "Relocated"
+  , "Integrity"
+  , "NX-Compatible"
+  , "NoIsolation"
+  , "NoSEH"
+  , "NoBind"
+  , "AppContainer"
+  , "WDM"
+  , "GuardCF"
+  , "TerminalServerAware"
 };
 
-static const CUInt32PCharPair g_SectFlags[] =
+
+// IMAGE_SCN_* constants:
+
+static const char * const g_SectFlags[] =
 {
-  {  3, "NoPad" },
-  {  5, "Code" },
-  {  6, "InitializedData" },
-  {  7, "UninitializedData" },
-  {  9, "Comments" },
-  { 11, "Remove" },
-  { 12, "COMDAT" },
-  { 15, "GP" },
-  { 24, "ExtendedRelocations" },
-  { 25, "Discardable" },
-  { 26, "NotCached" },
-  { 27, "NotPaged" },
-  { 28, "Shared" },
-  { 29, "Execute" },
-  { 30, "Read" },
-  { 31, "Write" }
+    NULL
+  , NULL
+  , NULL
+  , "NoPad"
+  , NULL
+  , "Code"
+  , "InitializedData"
+  , "UninitializedData"
+  , "Other"
+  , "Comments"
+  , NULL // OVER
+  , "Remove"
+  , "COMDAT"
+  , NULL
+  , "NO_DEFER_SPEC_EXC"
+  , "GP" // MEM_FARDATA
+  , NULL // SYSHEAP
+  , "PURGEABLE" // 16BIT
+  , "LOCKED"
+  , "PRELOAD"
+  , NULL
+  , NULL
+  , NULL
+  , NULL
+  , "ExtendedRelocations"
+  , "Discardable"
+  , "NotCached"
+  , "NotPaged"
+  , "Shared"
+  , "Execute"
+  , "Read"
+  , "Write"
 };
 
 static const CUInt32PCharPair g_MachinePairs[] =
@@ -699,7 +736,6 @@ class CHandler:
 {
   CMyComPtr<IInStream> _stream;
   CObjectVector<CSection> _sections;
-  UInt32 _peOffset;
   CHeader _header;
   UInt32 _totalSize;
   Int32 _mainSubfile;
@@ -720,9 +756,12 @@ class CHandler:
   bool _parseResources;
   bool _checksumError;
 
+  bool IsOpt() const { return _header.OptHeaderSize != 0; }
+
   COptHeader _optHeader;
 
   bool _allowTail;
+  bool _coffMode;
 
   HRESULT LoadDebugSections(IInStream *stream, bool &thereIsSection);
   HRESULT Open2(IInStream *stream, IArchiveOpenCallback *callback);
@@ -742,7 +781,10 @@ class CHandler:
   }
 
 public:
-  CHandler(): _allowTail(false) {}
+  CHandler(bool coffMode = false):
+        _coffMode(coffMode),
+        _allowTail(coffMode)
+        {}
 
   MY_UNKNOWN_IMP3(IInArchive, IInArchiveGetStream, IArchiveAllowTail)
   INTERFACE_IInArchive(;)
@@ -841,6 +883,34 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
   NCOM::CPropVariant prop;
   switch (propID)
   {
+    case kpidPhySize: prop = _totalSize; break;
+    case kpidComment: if (!_versionFullString.IsEmpty()) prop = _versionFullString; break;
+    case kpidShortComment:
+      if (!_versionShortString.IsEmpty())
+        prop = _versionShortString;
+      else
+      {
+        PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop);
+      }
+      break;
+
+    case kpidName: if (!_originalFilename.IsEmpty()) prop = _originalFilename; break;
+      
+    // case kpidIsSelfExe: prop = !_header.IsDll(); break;
+    // case kpidError:
+    case kpidWarning: if (_checksumError) prop = "Checksum error"; break;
+
+    case kpidCpu: PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); break;
+    case kpidMTime:
+    case kpidCTime: TimeToProp(_header.Time, prop); break;
+    case kpidCharacts: FLAGS_TO_PROP(g_HeaderCharacts, _header.Flags, prop); break;
+    case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break;
+    
+    default:
+    if (IsOpt())
+    switch (propID)
+    {
+
     case kpidSectAlign: prop = _optHeader.SectAlign; break;
     case kpidFileAlign: prop = _optHeader.FileAlign; break;
     case kpidLinkerVer:
@@ -857,37 +927,19 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
     case kpidInitDataSize: prop = _optHeader.InitDataSize; break;
     case kpidUnInitDataSize: prop = _optHeader.UninitDataSize; break;
     case kpidImageSize: prop = _optHeader.ImageSize; break;
-    case kpidPhySize: prop = _totalSize; break;
     case kpidHeadersSize: prop = _optHeader.HeadersSize; break;
     case kpidChecksum: prop = _optHeader.CheckSum; break;
-    case kpidComment: if (!_versionFullString.IsEmpty()) prop = _versionFullString; break;
-    case kpidShortComment:
-      if (!_versionShortString.IsEmpty())
-        prop = _versionShortString;
-      else
-      {
-        PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop);
-      }
-      break;
 
-    case kpidName: if (!_originalFilename.IsEmpty()) prop = _originalFilename; break;
     case kpidExtension:
       if (_header.IsDll())
-        prop = _optHeader.IsSybSystem_EFI() ? "efi" : "dll";
+        prop = "dll";
+      else if (_optHeader.IsSybSystem_EFI())
+        prop = "efi";
       break;
-      
-    // case kpidIsSelfExe: prop = !_header.IsDll(); break;
 
-    // case kpidError:
-    case kpidWarning: if (_checksumError) prop = "Checksum error"; break;
-
-    case kpidCpu: PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); break;
     case kpidBit64: if (_optHeader.Is64Bit()) prop = true; break;
     case kpidSubSystem: TYPE_TO_PROP(g_SubSystems, _optHeader.SubSystem, prop); break;
 
-    case kpidMTime:
-    case kpidCTime: TimeToProp(_header.Time, prop); break;
-    case kpidCharacts: FLAGS_TO_PROP(g_HeaderCharacts, _header.Flags, prop); break;
     case kpidDllCharacts: FLAGS_TO_PROP(g_DllCharacts, _optHeader.DllCharacts, prop); break;
     case kpidStackReserve: prop = _optHeader.StackReserve; break;
     case kpidStackCommit: prop = _optHeader.StackCommit; break;
@@ -898,8 +950,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
     // case kpidAddressOfEntryPoint: prop = _optHeader.AddressOfEntryPoint; break;
     // case kpidBaseOfCode: prop = _optHeader.BaseOfCode; break;
     // case kpidBaseOfData32: if (!_optHeader.Is64Bit()) prop = _optHeader.BaseOfData32; break;
-
-    case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break;
+  }
   }
   prop.Detach(value);
   return S_OK;
@@ -1056,7 +1107,24 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
       case kpidMTime:
       case kpidCTime:
         TimeToProp(item.IsDebug ? item.Time : _header.Time, prop); break;
-      case kpidCharacts: if (item.IsRealSect) FLAGS_TO_PROP(g_SectFlags, item.Flags, prop); break;
+      case kpidCharacts:
+       if (item.IsRealSect)
+       {
+         UInt32 flags = item.Flags;
+         const UInt32 MY__IMAGE_SCN_ALIGN_MASK = 0x00F00000;
+         AString s = FlagsToString(g_SectFlags, ARRAY_SIZE(g_SectFlags), item.Flags & ~MY__IMAGE_SCN_ALIGN_MASK);
+         const UInt32 align = ((flags >> 20) & 0xF);
+         if (align != 0)
+         {
+           char sz[32];
+           ConvertUInt32ToString(1 << (align - 1), sz);
+           s.Add_Space();
+           s += "align_";
+           s += sz;
+         }
+         prop = s;
+       }
+       break;
       case kpidZerosTailIsAllowed: if (!item.IsRealSect) prop = true; break;
     }
   }
@@ -1073,8 +1141,17 @@ HRESULT CHandler::LoadDebugSections(IInStream *stream, bool &thereIsSection)
     return S_OK;
   const unsigned kEntrySize = 28;
   UInt32 numItems = debugLink.Size / kEntrySize;
-  if (numItems * kEntrySize != debugLink.Size || numItems > 16)
+  if (numItems > 16)
     return S_FALSE;
+
+  // MAC's EFI file: numItems can be incorrect. Only first CDebugEntry entry is correct.
+  // debugLink.Size = kEntrySize + some_data, pointed by entry[0].
+  if (numItems * kEntrySize != debugLink.Size)
+  {
+    // return S_FALSE;
+    if (numItems > 1)
+      numItems = 1;
+  }
   
   UInt64 pa = 0;
   unsigned i;
@@ -2114,6 +2191,26 @@ HRESULT CHandler::OpenResources(unsigned sectionIndex, IInStream *stream, IArchi
   return S_OK;
 }
 
+
+bool CHeader::ParseCoff(const Byte *p)
+{
+  ParseBase(p);
+  if (PointerToSymbolTable < kCoffHeaderSize)
+    return false;
+  if (NumSymbols >= (1 << 24))
+    return false;
+  if (OptHeaderSize != 0 && OptHeaderSize < k_OptHeader32_Size_MIN)
+    return false;
+  for (unsigned i = 0; i < ARRAY_SIZE(g_MachinePairs); i++)
+    if (Machine == g_MachinePairs[i].Value)
+      return true;
+  if (Machine == 0)
+    return true;
+
+  return false;
+}
+
+
 static inline bool CheckPeOffset(UInt32 pe)
 {
   // ((pe & 7) == 0) is for most PE files. But there is unusual EFI-PE file that uses unaligned pe value.
@@ -2133,10 +2230,10 @@ API_FUNC_static_IsArc IsArc_Pe(const Byte *p, size_t size)
   UInt32 pe = Get32(p + 0x3C);
   if (!CheckPeOffset(pe))
     return k_IsArc_Res_NO;
-  if (pe + kHeaderSize > size)
+  if (pe + kPeHeaderSize > size)
     return k_IsArc_Res_NEED_MORE;
   CHeader header;
-  if (!header.Parse(p + pe))
+  if (!header.ParsePe(p + pe))
     return k_IsArc_Res_NO;
   return k_IsArc_Res_YES;
 }
@@ -2144,32 +2241,47 @@ API_FUNC_static_IsArc IsArc_Pe(const Byte *p, size_t size)
 
 HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
 {
+  UInt32 coffOffset = 0;
+  if (_coffMode)
   {
-    Byte h[kStartSize];
-    _mainSubfile = -1;
-    RINOK(ReadStream_FALSE(stream, h, kStartSize));
-    if (h[0] != 'M' || h[1] != 'Z')
-      return S_FALSE;
-    /* most of PE files contain 0x0090 at offset 2.
-       But some rare PE files contain another values. So we don't use that check.
-       if (Get16(h + 2) != 0x90) return false; */
-    _peOffset = Get32(h + 0x3C);
-    if (!CheckPeOffset(_peOffset))
+    Byte h[kCoffHeaderSize];
+    RINOK(ReadStream_FALSE(stream, h, kCoffHeaderSize));
+    if (!_header.ParseCoff(h))
       return S_FALSE;
   }
+  else
   {
-    Byte h[kHeaderSize];
-    RINOK(stream->Seek(_peOffset, STREAM_SEEK_SET, NULL));
-    RINOK(ReadStream_FALSE(stream, h, kHeaderSize));
-    if (!_header.Parse(h))
-      return S_FALSE;
+    UInt32 _peOffset;
+    {
+      Byte h[kStartSize];
+      RINOK(ReadStream_FALSE(stream, h, kStartSize));
+      if (h[0] != 'M' || h[1] != 'Z')
+        return S_FALSE;
+        /* most of PE files contain 0x0090 at offset 2.
+        But some rare PE files contain another values. So we don't use that check.
+      if (Get16(h + 2) != 0x90) return false; */
+      _peOffset = Get32(h + 0x3C);
+      if (!CheckPeOffset(_peOffset))
+        return S_FALSE;
+      coffOffset = _peOffset + 4;
+    }
+    {
+      Byte h[kPeHeaderSize];
+      RINOK(stream->Seek(_peOffset, STREAM_SEEK_SET, NULL));
+      RINOK(ReadStream_FALSE(stream, h, kPeHeaderSize));
+      if (!_header.ParsePe(h))
+        return S_FALSE;
+    }
   }
 
-  UInt32 bufSize = _header.OptHeaderSize + (UInt32)_header.NumSections * kSectionSize;
-  _totalSize = _peOffset + kHeaderSize + bufSize;
+  const UInt32 optStart = coffOffset + kCoffHeaderSize;
+  const UInt32 bufSize = _header.OptHeaderSize + (UInt32)_header.NumSections * kSectionSize;
+  _totalSize = optStart + bufSize;
   CByteBuffer buffer(bufSize);
 
   RINOK(ReadStream_FALSE(stream, buffer, bufSize));
+  
+  if (_header.OptHeaderSize != 0)
   if (!_optHeader.Parse(buffer, _header.OptHeaderSize))
     return S_FALSE;
 
@@ -2207,7 +2319,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
   for (i = 0; i < _sections.Size(); i++)
     _sections[i].UpdateTotalSize(_totalSize);
 
-  bool thereISDebug;
+  bool thereISDebug = false;
+  if (IsOpt())
+  {
   RINOK(LoadDebugSections(stream, thereISDebug));
 
   const CDirLink &certLink = _optHeader.DirItems[kDirLink_Certificate];
@@ -2256,8 +2370,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
         _totalSize += (UInt32)k;
     }
   }
+  }
 
-  if (_header.NumSymbols > 0 && _header.PointerToSymbolTable >= 512)
+  if (_header.NumSymbols > 0 && _header.PointerToSymbolTable >= optStart)
   {
     if (_header.NumSymbols >= (1 << 24))
       return S_FALSE;
@@ -2306,11 +2421,12 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
   }
 
 
+  if (IsOpt())
   if (_optHeader.CheckSum != 0)
   {
     RINOK(stream->Seek(0, STREAM_SEEK_SET, NULL));
     UInt32 checkSum = 0;
-    RINOK(CalcCheckSum(stream, _totalSize, _peOffset + kHeaderSize + k_CheckSum_Field_Offset, checkSum));
+    RINOK(CalcCheckSum(stream, _totalSize, optStart + k_CheckSum_Field_Offset, checkSum));
     _checksumError = (checkSum != _optHeader.CheckSum);
   }
 
@@ -2333,6 +2449,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
     const CSection &sect = _sections[i];
     CMixItem mixItem;
     mixItem.SectionIndex = i;
+    if (IsOpt())
     if (_parseResources && sect.Name == ".rsrc" && _items.IsEmpty())
     {
       const unsigned numMixItems = _mixItems.Size();
@@ -2480,6 +2597,8 @@ STDMETHODIMP CHandler::Close()
 {
   _totalSize = 0;
   _checksumError = false;
+  _mainSubfile = -1;
+
   _stream.Release();
   _sections.Clear();
   _mixItems.Clear();
@@ -2675,10 +2794,41 @@ REGISTER_ARC_I(
   0,
   NArcInfoFlags::kPreArc,
   IsArc_Pe)
- 
+
 }
 
+namespace NCoff {
 
+API_FUNC_static_IsArc IsArc_Coff(const Byte *p, size_t size)
+{
+  if (size < NPe::kCoffHeaderSize)
+    return k_IsArc_Res_NEED_MORE;
+  NPe::CHeader header;
+  if (!header.ParseCoff(p))
+    return k_IsArc_Res_NO;
+  return k_IsArc_Res_YES;
+}
+}
+
+/*
+static const Byte k_Signature[] =
+{
+    2, 0x4C, 0x01, // x86
+    2, 0x64, 0x86, // x64
+    2, 0x64, 0xAA  // ARM64
+};
+REGISTER_ARC_I_CLS(
+*/
+
+REGISTER_ARC_I_CLS_NO_SIG(
+  NPe::CHandler(true),
+  "COFF", "obj", 0, 0xC6,
+  // k_Signature,
+  0,
+  // NArcInfoFlags::kMultiSignature |
+  NArcInfoFlags::kStartOpen,
+  IsArc_Coff)
+}
 
 
 namespace NTe {

CPP/7zip/Archive/Rar/Rar5Handler.cpp

@@ -1086,7 +1086,7 @@ HRESULT CUnpacker::Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool
     if (!lzCoder)
     {
       const UInt32 methodID = 0x40305;
-      RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder));
+      RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodID, false, lzCoder));
       if (!lzCoder)
         return E_NOTIMPL;
     }
@@ -1363,7 +1363,7 @@ static const Byte kProps[] =
   kpidCharacts,
   kpidSymLink,
   kpidHardLink,
-  kpidCopyLink,
+  kpidCopyLink
 };
 
 

CPP/7zip/Archive/Rar/RarHandler.cpp

@@ -1691,7 +1691,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
               methodID += 2;
             else
               methodID += 3;
-            RINOK(CreateCoder(EXTERNAL_CODECS_VARS methodID, false, mi.Coder));
+            RINOK(CreateCoder_Id(EXTERNAL_CODECS_VARS methodID, false, mi.Coder));
           }
          
           if (mi.Coder == 0)

CPP/7zip/Archive/SquashfsHandler.cpp

@@ -2,7 +2,6 @@
 
 #include "StdAfx.h"
 
-#include "../../../C/7zCrc.h"
 #include "../../../C/Alloc.h"
 #include "../../../C/CpuArch.h"
 #include "../../../C/Xz.h"
@@ -60,6 +59,7 @@ UInt64 Get64b(const Byte *p, bool be) { return be ? GetBe64(p) : GetUi64(p); }
 static const UInt32 kSignature32_LE = 0x73717368;
 static const UInt32 kSignature32_BE = 0x68737173;
 static const UInt32 kSignature32_LZ = 0x71736873;
+static const UInt32 kSignature32_B2 = 0x73687371;
 
 #define kMethod_ZLIB 1
 #define kMethod_LZMA 2
@@ -225,6 +225,7 @@ struct CHeader
       case kSignature32_LE: break;
       case kSignature32_BE: be = true; break;
       case kSignature32_LZ: SeveralMethods = true; break;
+      case kSignature32_B2: SeveralMethods = true; be = true; break;
       default: return false;
     }
     GET_32 (4, NumInodes);
@@ -1207,8 +1208,10 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool
     else
     {
       ECoderStatus status;
-      XzUnpacker_Init(&_xz);
-      SRes res = XzUnpacker_Code(&_xz, dest, &destLen, _inputBuffer, &srcLen, CODER_FINISH_END, &status);
+      SRes res = XzUnpacker_CodeFull(&_xz,
+          dest, &destLen,
+          _inputBuffer, &srcLen,
+          CODER_FINISH_END, &status);
       if (res != 0)
         return SResToHRESULT(res);
       if (status != CODER_STATUS_NEEDS_MORE_INPUT || !XzUnpacker_IsStreamWasFinished(&_xz))
@@ -2258,7 +2261,8 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream)
 static const Byte k_Signature[] = {
     4, 'h', 's', 'q', 's',
     4, 's', 'q', 's', 'h',
-    4, 's', 'h', 's', 'q' };
+    4, 's', 'h', 's', 'q',
+    4, 'q', 's', 'h', 's' };
 
 REGISTER_ARC_I(
   "SquashFS", "squashfs", 0, 0xD2,

CPP/7zip/Archive/Tar/TarIn.cpp

@@ -26,24 +26,25 @@ static void MyStrNCpy(char *dest, const char *src, unsigned size)
   }
 }
 
-static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res)
+static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res, bool allowEmpty = false)
 {
+  res = 0;
   char sz[32];
   MyStrNCpy(sz, srcString, size);
   sz[size] = 0;
   const char *end;
   unsigned i;
   for (i = 0; sz[i] == ' '; i++);
+  if (sz[i] == 0)
+    return allowEmpty;
   res = ConvertOctStringToUInt64(sz + i, &end);
-  if (end == sz + i)
-    return false;
   return (*end == ' ' || *end == 0);
 }
 
-static bool OctalToNumber32(const char *srcString, unsigned size, UInt32 &res)
+static bool OctalToNumber32(const char *srcString, unsigned size, UInt32 &res, bool allowEmpty = false)
 {
   UInt64 res64;
-  if (!OctalToNumber(srcString, size, res64))
+  if (!OctalToNumber(srcString, size, res64, allowEmpty))
     return false;
   res = (UInt32)res64;
   return (res64 <= 0xFFFFFFFF);
@@ -123,7 +124,8 @@ API_FUNC_IsArc IsArc_Tar(const Byte *p2, size_t size)
   p += NFileHeader::kNameSize;
 
   UInt32 mode;
-  CHECK(OctalToNumber32(p, 8, mode)); p += 8;
+  // we allow empty Mode value for LongName prefix items
+  CHECK(OctalToNumber32(p, 8, mode, true)); p += 8;
 
   // if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0;
   p += 8;
@@ -194,7 +196,8 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE
       (item.Name.Len() == NFileHeader::kNameSize ||
        item.Name.Len() == NFileHeader::kNameSize - 1);
 
-  RIF(OctalToNumber32(p, 8, item.Mode)); p += 8;
+  // we allow empty Mode value for LongName prefix items
+  RIF(OctalToNumber32(p, 8, item.Mode, true)); p += 8;
 
   if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0; p += 8;
   if (!OctalToNumber32(p, 8, item.GID)) item.GID = 0; p += 8;

CPP/7zip/Archive/Zip/ZipAddCommon.cpp

@@ -36,6 +36,7 @@ static const UInt32 kLzmaHeaderSize = 4 + kLzmaPropsSize;
 class CLzmaEncoder:
   public ICompressCoder,
   public ICompressSetCoderProperties,
+  public ICompressSetCoderPropertiesOpt,
   public CMyUnknownImp
 {
 public:
@@ -46,8 +47,11 @@ public:
   STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
       const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
   STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
+  STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
 
-  MY_UNKNOWN_IMP1(ICompressSetCoderProperties)
+  MY_UNKNOWN_IMP2(
+      ICompressSetCoderProperties,
+      ICompressSetCoderPropertiesOpt)
 };
 
 STDMETHODIMP CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)
@@ -71,6 +75,11 @@ STDMETHODIMP CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPV
   return S_OK;
 }
 
+STDMETHODIMP CLzmaEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)
+{
+  return EncoderSpec->SetCoderPropertiesOpt(propIDs, props, numProps);
+}
+
 STDMETHODIMP CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
     const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)
 {
@@ -118,7 +127,8 @@ HRESULT CAddCommon::CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultC
 }
 
 
-HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, CCompressingResult &opRes) const
+HRESULT CAddCommon::Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize,
+    CCompressingResult &opRes) const
 {
   // We use Zip64, if unPackSize size is larger than 0xF8000000 to support
   // cases when compressed size can be about 3% larger than uncompressed size
@@ -144,7 +154,7 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C
   opRes.LzmaEos = false;
 
   opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default;
-  opRes.FileTimeWasUsed = false;
+  opRes.DescriptorMode = outSeqMode;
   
   if (_options.PasswordIsDefined)
   {
@@ -153,8 +163,8 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C
       opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Aes;
     else
     {
-      if (seqMode)
-        opRes.FileTimeWasUsed = true;
+      if (inSeqMode)
+        opRes.DescriptorMode = true;
     }
   }
   
@@ -187,7 +197,8 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C
 HRESULT CAddCommon::Compress(
     DECL_EXTERNAL_CODECS_LOC_VARS
     ISequentialInStream *inStream, IOutStream *outStream,
-    bool seqMode, UInt32 fileTime,
+    bool inSeqMode, bool outSeqMode,
+    UInt32 fileTime, UInt64 expectedDataSize,
     ICompressProgressInfo *progress, CCompressingResult &opRes)
 {
   opRes.LzmaEos = false;
@@ -202,16 +213,31 @@ HRESULT CAddCommon::Compress(
   CMyComPtr<ISequentialInStream> inCrcStream = inSecCrcStreamSpec;
   
   CMyComPtr<IInStream> inStream2;
-  if (!seqMode)
+  if (!inSeqMode)
+  {
     inStream->QueryInterface(IID_IInStream, (void **)&inStream2);
+    if (!inStream2)
+    {
+      // inSeqMode = true;
+      // inSeqMode must be correct before
+      return E_FAIL;
+    }
+  }
 
   inSecCrcStreamSpec->SetStream(inStream);
   inSecCrcStreamSpec->Init();
 
   unsigned numTestMethods = _options.MethodSequence.Size();
-  
-  if (seqMode || (numTestMethods > 1 && !inStream2))
-    numTestMethods = 1;
+
+  bool descriptorMode = outSeqMode;
+  if (!outSeqMode)
+    if (inSeqMode && _options.PasswordIsDefined && !_options.IsAesMode)
+      descriptorMode = true;
+  opRes.DescriptorMode = descriptorMode;
+
+  if (numTestMethods > 1)
+    if (inSeqMode || outSeqMode || !inStream2)
+      numTestMethods = 1;
 
   UInt32 crc = 0;
   bool crc_IsCalculated = false;
@@ -219,19 +245,23 @@ HRESULT CAddCommon::Compress(
   Byte method = 0;
   CFilterCoder::C_OutStream_Releaser outStreamReleaser;
   opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default;
-  opRes.FileTimeWasUsed = false;
   
   for (unsigned i = 0; i < numTestMethods; i++)
   {
     opRes.LzmaEos = false;
     opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default;
-    if (inStream2 && i != 0)
+    
+    if (i != 0)
     {
-      inSecCrcStreamSpec->Init();
-      RINOK(inStream2->Seek(0, STREAM_SEEK_SET, NULL));
+      if (inStream2)
+      {
+        inSecCrcStreamSpec->Init();
+        RINOK(inStream2->Seek(0, STREAM_SEEK_SET, NULL));
+      }
+
+      RINOK(outStream->SetSize(0));
+      RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL));
     }
-    RINOK(outStream->SetSize(0));
-    RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL));
     
     if (_options.PasswordIsDefined)
     {
@@ -264,7 +294,12 @@ HRESULT CAddCommon::Compress(
         
         UInt32 check;
         
-        if (inStream2)
+        if (descriptorMode)
+        {
+          // it's Info-ZIP modification for stream_mode descriptor_mode (bit 3 of the general purpose bit flag is set)
+          check = (fileTime & 0xFFFF);
+        }
+        else
         {
           if (!crc_IsCalculated)
           {
@@ -275,11 +310,6 @@ HRESULT CAddCommon::Compress(
           }
           check = (crc >> 16);
         }
-        else
-        {
-          opRes.FileTimeWasUsed = true;
-          check = (fileTime & 0xFFFF);
-        }
         
         RINOK(_filterSpec->WriteHeader_Check16(outStream, (UInt16)check));
       }
@@ -295,7 +325,13 @@ HRESULT CAddCommon::Compress(
     {
       case NCompressionMethod::kStore:
       {
-        if (_copyCoderSpec == NULL)
+        if (descriptorMode)
+        {
+          // we still can create descriptor_mode archives with "Store" method, but they are not good for 100%
+          return E_NOTIMPL;
+        }
+
+        if (!_copyCoderSpec)
         {
           _copyCoderSpec = new NCompress::CCopyCoder;
           _copyCoder = _copyCoderSpec;
@@ -348,7 +384,7 @@ HRESULT CAddCommon::Compress(
               methodId = kMethodId_ZipBase + method;
               break;
           }
-          RINOK(CreateCoder(
+          RINOK(CreateCoder_Id(
               EXTERNAL_CODECS_LOC_VARS
               methodId, true, _compressEncoder));
           if (!_compressEncoder)
@@ -390,6 +426,18 @@ HRESULT CAddCommon::Compress(
           outStreamNew = outStream;
         if (_compressExtractVersion > opRes.ExtractVersion)
           opRes.ExtractVersion = _compressExtractVersion;
+
+        {
+          CMyComPtr<ICompressSetCoderPropertiesOpt> optProps;
+          _compressEncoder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps);
+          if (optProps)
+          {
+            PROPID propID = NCoderPropID::kExpectedDataSize;
+            NWindows::NCOM::CPropVariant prop = (UInt64)expectedDataSize;
+            RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1));
+          }
+        }
+        
         RINOK(_compressEncoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress));
         break;
       }

CPP/7zip/Archive/Zip/ZipAddCommon.h

@@ -26,7 +26,7 @@ struct CCompressingResult
   UInt32 CRC;
   UInt16 Method;
   Byte ExtractVersion;
-  bool FileTimeWasUsed;
+  bool DescriptorMode;
   bool LzmaEos;
 };
 
@@ -53,12 +53,14 @@ public:
   CAddCommon(const CCompressionMethodMode &options);
   ~CAddCommon();
 
-  HRESULT Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, CCompressingResult &opRes) const;
+  HRESULT Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize,
+      CCompressingResult &opRes) const;
   
   HRESULT Compress(
       DECL_EXTERNAL_CODECS_LOC_VARS
       ISequentialInStream *inStream, IOutStream *outStream,
-      bool seqMode, UInt32 fileTime,
+      bool inSeqMode, bool outSeqMode,
+      UInt32 fileTime, UInt64 expectedDataSize,
       ICompressProgressInfo *progress, CCompressingResult &opRes);
 };
 

CPP/7zip/Archive/Zip/ZipHandler.cpp

@@ -112,7 +112,8 @@ static const CUInt32PCharPair g_HeaderCharacts[] =
   { 3, "Descriptor" },
   // { 5, "Patched" },
   { 6, kMethod_StrongCrypto },
-  { 11, "UTF8" }
+  { 11, "UTF8" },
+  { 14, "Alt" }
 };
 
 struct CIdToNamePair
@@ -169,6 +170,7 @@ static const Byte kProps[] =
   kpidUnpackVer,
   kpidVolumeIndex,
   kpidOffset
+  // kpidIsAltStream
 };
 
 static const Byte kArcProps[] =
@@ -307,6 +309,8 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
           prop = true;
       break;
     }
+
+    // case kpidIsAltStream: prop = true; break;
   }
   prop.Detach(value);
   COM_TRY_END
@@ -333,6 +337,21 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
       UString res;
       item.GetUnicodeString(res, item.Name, false, _forceCodePage, _specifiedCodePage);
       NItemName::ReplaceToOsSlashes_Remove_TailSlash(res);
+      /*
+      if (item.ParentOfAltStream >= 0)
+      {
+        const CItemEx &prevItem = m_Items[item.ParentOfAltStream];
+        UString prevName;
+        prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage);
+        NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName);
+        if (res.IsPrefixedBy(prevName))
+          if (IsString1PrefixedByString2(res.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM))
+          {
+            res.Delete(prevName.Len(), (unsigned)strlen(k_SpecName_NTFS_STREAM));
+            res.Insert(prevName.Len(), L":");
+          }
+      }
+      */
       prop = res;
       break;
     }
@@ -564,7 +583,8 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
         AString s2 = FlagsToString(g_HeaderCharacts, ARRAY_SIZE(g_HeaderCharacts), flags);
         if (!s2.IsEmpty())
         {
-          s.Add_OptSpaced(":");
+          if (!s.IsEmpty())
+            s.Add_OptSpaced(":");
           s.Add_OptSpaced(s2);
         }
       }
@@ -595,6 +615,19 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
     case kpidOffset:
       prop = item.LocalHeaderPos;
       break;
+
+    /*
+    case kpidIsAltStream:
+      prop = (bool)(item.ParentOfAltStream >= 0); // item.IsAltStream();
+      break;
+
+    case kpidName:
+      if (item.ParentOfAltStream >= 0)
+      {
+        // extract name of stream here
+      }
+      break;
+    */
   }
   
   prop.Detach(value);
@@ -603,6 +636,85 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
 }
 
 
+
+/*
+STDMETHODIMP CHandler::GetNumRawProps(UInt32 *numProps)
+{
+  *numProps = 0;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID)
+{
+  UNUSED_VAR(index);
+  *propID = 0;
+  *name = 0;
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetParent(UInt32 index, UInt32 *parent, UInt32 *parentType)
+{
+  *parentType = NParentType::kDir;
+  *parent = (UInt32)(Int32)-1;
+  if (index >= m_Items.Size())
+    return S_OK;
+  const CItemEx &item = m_Items[index];
+
+  if (item.ParentOfAltStream >= 0)
+  {
+    *parentType = NParentType::kAltStream;
+    *parent = item.ParentOfAltStream;
+  }
+  return S_OK;
+}
+
+STDMETHODIMP CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType)
+{
+  UNUSED_VAR(index);
+  UNUSED_VAR(propID);
+  *data = NULL;
+  *dataSize = 0;
+  *propType = 0;
+  return S_OK;
+}
+
+
+void CHandler::MarkAltStreams(CObjectVector<CItemEx> &items)
+{
+  int prevIndex = -1;
+  UString prevName;
+  UString name;
+
+  for (unsigned i = 0; i < items.Size(); i++)
+  {
+    CItemEx &item = m_Items[i];
+    if (item.IsAltStream())
+    {
+      if (prevIndex == -1)
+        continue;
+      if (prevName.IsEmpty())
+      {
+        const CItemEx &prevItem = m_Items[prevIndex];
+        prevItem.GetUnicodeString(prevName, prevItem.Name, false, _forceCodePage, _specifiedCodePage);
+        NItemName::ReplaceToOsSlashes_Remove_TailSlash(prevName);
+      }
+      name.Empty();
+      item.GetUnicodeString(name, item.Name, false, _forceCodePage, _specifiedCodePage);
+      NItemName::ReplaceToOsSlashes_Remove_TailSlash(name);
+      
+      if (name.IsPrefixedBy(prevName))
+        if (IsString1PrefixedByString2(name.Ptr(prevName.Len()), k_SpecName_NTFS_STREAM))
+          item.ParentOfAltStream = prevIndex;
+    }
+    else
+    {
+      prevIndex = i;
+      prevName.Empty();
+    }
+  }
+}
+*/
+
 STDMETHODIMP CHandler::Open(IInStream *inStream,
     const UInt64 *maxCheckStartPosition, IArchiveOpenCallback *callback)
 {
@@ -616,6 +728,7 @@ STDMETHODIMP CHandler::Open(IInStream *inStream,
       m_Items.Clear();
       m_Archive.ClearRefs(); // we don't want to clear error flags
     }
+    // MarkAltStreams(m_Items);
     return res;
   }
   catch(...) { Close(); throw; }
@@ -737,7 +850,7 @@ public:
     IArchiveExtractCallback *extractCallback,
     ICompressProgressInfo *compressProgress,
     #ifndef _7ZIP_ST
-    UInt32 numThreads,
+    UInt32 numThreads, UInt64 memUsage,
     #endif
     Int32 &res);
 };
@@ -766,7 +879,7 @@ HRESULT CZipDecoder::Decode(
     IArchiveExtractCallback *extractCallback,
     ICompressProgressInfo *compressProgress,
     #ifndef _7ZIP_ST
-    UInt32 numThreads,
+    UInt32 numThreads, UInt64 memUsage,
     #endif
     Int32 &res)
 {
@@ -961,7 +1074,7 @@ HRESULT CZipDecoder::Decode(
         szMethodID = kMethodId_ZipBase + (Byte)id;
       }
 
-      RINOK(CreateCoder(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder));
+      RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS szMethodID, false, mi.Coder));
 
       if (!mi.Coder)
       {
@@ -973,16 +1086,7 @@ HRESULT CZipDecoder::Decode(
   }
 
   ICompressCoder *coder = methodItems[m].Coder;
-  
-  {
-    CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties;
-    coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties);
-    if (setDecoderProperties)
-    {
-      Byte properties = (Byte)item.Flags;
-      RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1));
-    }
-  }
+
   
   #ifndef _7ZIP_ST
   {
@@ -993,7 +1097,27 @@ HRESULT CZipDecoder::Decode(
       RINOK(setCoderMt->SetNumberOfThreads(numThreads));
     }
   }
+  // if (memUsage != 0)
+  {
+    CMyComPtr<ICompressSetMemLimit> setMemLimit;
+    coder->QueryInterface(IID_ICompressSetMemLimit, (void **)&setMemLimit);
+    if (setMemLimit)
+    {
+      RINOK(setMemLimit->SetMemLimit(memUsage));
+    }
+  }
   #endif
+
+  {
+    CMyComPtr<ICompressSetDecoderProperties2> setDecoderProperties;
+    coder->QueryInterface(IID_ICompressSetDecoderProperties2, (void **)&setDecoderProperties);
+    if (setDecoderProperties)
+    {
+      Byte properties = (Byte)item.Flags;
+      RINOK(setDecoderProperties->SetDecoderProperties2(&properties, 1));
+    }
+  }
+  
   
   CMyComPtr<ISequentialInStream> inStreamNew;
 
@@ -1318,7 +1442,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
         m_Archive, item, realOutStream, extractCallback,
         progress,
         #ifndef _7ZIP_ST
-        _props._numThreads,
+        _props._numThreads, _props._memUsage,
         #endif
         res);
     

CPP/7zip/Archive/Zip/ZipHandler.h

@@ -25,6 +25,7 @@ extern const char * const kMethodNames2[kNumMethodNames2];
 
 class CHandler:
   public IInArchive,
+  // public IArchiveGetRawProps,
   public IOutArchive,
   public ISetProperties,
   PUBLIC_ISetCompressCodecsInfo
@@ -32,6 +33,7 @@ class CHandler:
 {
 public:
   MY_QUERYINTERFACE_BEGIN2(IInArchive)
+  // MY_QUERYINTERFACE_ENTRY(IArchiveGetRawProps)
   MY_QUERYINTERFACE_ENTRY(IOutArchive)
   MY_QUERYINTERFACE_ENTRY(ISetProperties)
   QUERY_ENTRY_ISetCompressCodecsInfo
@@ -39,6 +41,7 @@ public:
   MY_ADDREF_RELEASE
 
   INTERFACE_IInArchive(;)
+  // INTERFACE_IArchiveGetRawProps(;)
   INTERFACE_IOutArchive(;)
 
   STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
@@ -75,6 +78,10 @@ private:
     _forceCodePage = false;
     _specifiedCodePage = CP_OEMCP;
   }
+
+  // void MarkAltStreams(CObjectVector<CItemEx> &items);
+
+  HRESULT GetOutProperty(IArchiveUpdateCallback *callback, UInt32 callbackIndex, Int32 arcIndex, PROPID propID, PROPVARIANT *value);
 };
 
 }}

CPP/7zip/Archive/Zip/ZipHandlerOut.cpp

@@ -136,6 +136,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
         thereAreAesUpdates = true;
       if (!IntToBool(newProps))
         ui.IsDir = inputItem.IsDir();
+      // ui.IsAltStream = inputItem.IsAltStream();
     }
 
     if (IntToBool(newProps))
@@ -175,6 +176,33 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
           ui.IsDir = (prop.boolVal != VARIANT_FALSE);
       }
 
+      /*
+      {
+        bool isAltStream = false;
+        {
+          NCOM::CPropVariant prop;
+          RINOK(callback->GetProperty(i, kpidIsAltStream, &prop));
+          if (prop.vt == VT_BOOL)
+            isAltStream = (prop.boolVal != VARIANT_FALSE);
+          else if (prop.vt != VT_EMPTY)
+            return E_INVALIDARG;
+        }
+      
+        if (isAltStream)
+        {
+          if (ui.IsDir)
+            return E_INVALIDARG;
+          int delim = name.ReverseFind(L':');
+          if (delim >= 0)
+          {
+            name.Delete(delim, 1);
+            name.Insert(delim, UString(k_SpecName_NTFS_STREAM));
+            ui.IsAltStream = true;
+          }
+        }
+      }
+      */
+
       {
         CPropVariant prop;
         RINOK(callback->GetProperty(i, kpidTimeType, &prop));
@@ -348,7 +376,8 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
         {
           CMethodId methodId;
           UInt32 numStreams;
-          if (!FindMethod(EXTERNAL_CODECS_VARS methodName, methodId, numStreams))
+          if (FindMethod_Index(EXTERNAL_CODECS_VARS methodName, true,
+              methodId, numStreams) < 0)
             return E_NOTIMPL;
           if (numStreams != 1)
             return E_NOTIMPL;

CPP/7zip/Archive/Zip/ZipHeader.h

@@ -131,6 +131,7 @@ namespace NFileHeader
     const unsigned kDescriptorUsedMask = 1 << 3;
     const unsigned kStrongEncrypted = 1 << 6;
     const unsigned kUtf8 = 1 << 11;
+    const unsigned kAltStream = 1 << 14;
 
     const unsigned kImplodeDictionarySizeMask = 1 << 1;
     const unsigned kImplodeLiteralsOnMask     = 1 << 2;

CPP/7zip/Archive/Zip/ZipIn.cpp

@@ -459,7 +459,16 @@ API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size)
       extraSize -= 4;
       p += 4;
       if (dataSize > extraSize)
-        return k_IsArc_Res_NO;
+      {
+        // It can be error on header.
+        // We want to support such rare case bad archives.
+        // We use additional checks to reduce false-positive probability.
+        if (nameSize == 0
+            || nameSize > (1 << 9)
+            || extraSize > (1 << 9))
+          return k_IsArc_Res_NO;
+        return k_IsArc_Res_YES;
+      }
       if (dataSize > size)
         return k_IsArc_Res_NEED_MORE;
       size -= dataSize;

CPP/7zip/Archive/Zip/ZipIn.h

@@ -20,10 +20,13 @@ class CItemEx: public CItem
 {
 public:
   UInt32 LocalFullHeaderSize; // including Name and Extra
+  // int ParentOfAltStream; // -1, if not AltStream
   
   bool DescriptorWasRead;
 
-  CItemEx(): DescriptorWasRead(false) {}
+  CItemEx():
+    // ParentOfAltStream(-1),
+    DescriptorWasRead(false) {}
 
   UInt64 GetLocalFullSize() const
     { return LocalFullHeaderSize + GetPackSizeWithDescriptor(); }

CPP/7zip/Archive/Zip/ZipItem.cpp

@@ -20,6 +20,12 @@ namespace NZip {
 
 using namespace NFileHeader;
 
+
+/*
+const char *k_SpecName_NTFS_STREAM = "@@NTFS@STREAM@";
+const char *k_SpecName_MAC_RESOURCE_FORK = "@@MAC@RESOURCE-FORK@";
+*/
+
 static const CUInt32PCharPair g_ExtraTypes[] =
 {
   { NExtraID::kZip64, "Zip64" },
@@ -269,7 +275,25 @@ UInt32 CItem::GetWinAttrib() const
     case NHostOS::kUnix:
       // do we need to clear 16 low bits in this case?
       if (FromCentral)
+      {
+        /*
+          Some programs write posix attributes in high 16 bits of ExternalAttrib
+          Also some programs can write additional marker flag:
+            0x8000 - p7zip
+            0x4000 - Zip in MacOS
+            no marker - Info-Zip
+
+          Client code has two options to detect posix field:
+            1) check 0x8000 marker. In that case we must add 0x8000 marker here.
+            2) check that high 4 bits (file type bits in posix field) of attributes are not zero.
+        */
+        
         winAttrib = ExternalAttrib & 0xFFFF0000;
+        
+        // #ifndef _WIN32
+        winAttrib |= 0x8000; // add posix mode marker
+        // #endif
+      }
       break;
   }
   if (IsDir()) // test it;

CPP/7zip/Archive/Zip/ZipItem.h

@@ -14,6 +14,11 @@
 namespace NArchive {
 namespace NZip {
 
+/*
+extern const char *k_SpecName_NTFS_STREAM;
+extern const char *k_SpecName_MAC_RESOURCE_FORK;
+*/
+
 struct CVersion
 {
   Byte Version;
@@ -233,6 +238,7 @@ public:
   bool IsAesEncrypted() const { return IsEncrypted() && (IsStrongEncrypted() || Method == NFileHeader::NCompressionMethod::kWzAES); }
   bool IsLzmaEOS() const { return (Flags & NFileHeader::NFlags::kLzmaEOS) != 0; }
   bool HasDescriptor() const { return (Flags & NFileHeader::NFlags::kDescriptorUsedMask) != 0; }
+  // bool IsAltStream() const { return (Flags & NFileHeader::NFlags::kAltStream) != 0; }
 
   unsigned GetDeflateLevel() const { return (Flags >> 1) & 3; }
   
@@ -264,6 +270,7 @@ public:
   void ClearFlags() { Flags = 0; }
   void SetEncrypted(bool encrypted) { SetFlag(NFileHeader::NFlags::kEncrypted, encrypted); }
   void SetUtf8(bool isUtf8) { SetFlag(NFileHeader::NFlags::kUtf8, isUtf8); }
+  // void SetFlag_AltStream(bool isAltStream) { SetFlag(NFileHeader::NFlags::kAltStream, isAltStream); }
   void SetDescriptorMode(bool useDescriptor) { SetFlag(NFileHeader::NFlags::kDescriptorUsedMask, useDescriptor); }
 
   UINT GetCodePage() const { return CP_OEMCP; }

CPP/7zip/Archive/Zip/ZipOut.cpp

@@ -174,6 +174,9 @@ void COutArchive::WriteLocalHeader_Replace(CItemOut &item)
   {
     WriteDescriptor(item);
     m_OutBuffer.FlushWithCheck();
+    return;
+    // we don't replace local header, if we write Descriptor.
+    // so local header with Descriptor flag must be written to local header before.
   }
 
   const UInt64 nextPos = m_CurPos;

CPP/7zip/Archive/Zip/ZipUpdate.cpp

@@ -62,7 +62,7 @@ static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method)
 static void SetFileHeader(
     const CCompressionMethodMode &options,
     const CUpdateItem &ui,
-    // bool isSeqMode,
+    bool useDescriptor,
     CItemOut &item)
 {
   item.Size = ui.Size;
@@ -75,6 +75,7 @@ static void SetFileHeader(
     item.Name = ui.Name;
     item.Comment = ui.Comment;
     item.SetUtf8(ui.IsUtf8);
+    // item.SetFlag_AltStream(ui.IsAltStream);
     item.ExternalAttrib = ui.Attrib;
     item.Time = ui.Time;
     item.Ntfs_MTime = ui.Ntfs_MTime;
@@ -94,7 +95,7 @@ static void SetFileHeader(
 
   item.InternalAttrib = 0; // test it
   item.SetEncrypted(!isDir && options.PasswordIsDefined);
-  // item.SetDescriptorMode(isSeqMode);
+  item.SetDescriptorMode(useDescriptor);
 
   if (isDir)
   {
@@ -166,18 +167,22 @@ struct CThreadInfo
   HRESULT Result;
   CCompressingResult CompressingResult;
 
-  bool SeqMode;
+  bool InSeqMode;
+  bool OutSeqMode;
   bool IsFree;
   UInt32 UpdateIndex;
   UInt32 FileTime;
+  UInt64 ExpectedDataSize;
 
   CThreadInfo(const CCompressionMethodMode &options):
       ExitThread(false),
       ProgressSpec(0),
       OutStreamSpec(0),
       Coder(options),
-      SeqMode(false),
-      FileTime(0)
+      InSeqMode(false),
+      OutSeqMode(false),
+      FileTime(0),
+      ExpectedDataSize((UInt64)(Int64)-1)
   {}
   
   HRESULT CreateEvents()
@@ -210,7 +215,9 @@ void CThreadInfo::WaitAndCode()
     
     Result = Coder.Compress(
         EXTERNAL_CODECS_LOC_VARS
-        InStream, OutStream, SeqMode, FileTime, Progress, CompressingResult);
+        InStream, OutStream,
+        InSeqMode, OutSeqMode, FileTime, ExpectedDataSize,
+        Progress, CompressingResult);
     
     if (Result == S_OK && Progress)
       Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize);
@@ -237,10 +244,13 @@ public:
 
 struct CMemBlocks2: public CMemLockBlocks
 {
-  CCompressingResult CompressingResult;
-  bool Defined;
   bool Skip;
-  CMemBlocks2(): Defined(false), Skip(false) {}
+  bool InSeqMode;
+  bool PreDescriptorMode;
+  bool Finished;
+  CCompressingResult CompressingResult;
+  
+  CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false) {}
 };
 
 class CMemRefs
@@ -271,6 +281,7 @@ public:
   MY_UNKNOWN_IMP
   void Create(IProgress *progress, bool inSizeIsMain);
   void SetProgressOffset(UInt64 progressOffset);
+  void SetProgressOffset_NoLock(UInt64 progressOffset);
   HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize);
   STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize);
 };
@@ -283,11 +294,16 @@ void CMtProgressMixer2::Create(IProgress *progress, bool inSizeIsMain)
   ProgressOffset = InSizes[0] = InSizes[1] = OutSizes[0] = OutSizes[1] = 0;
 }
 
+void CMtProgressMixer2::SetProgressOffset_NoLock(UInt64 progressOffset)
+{
+  InSizes[1] = OutSizes[1] = 0;
+  ProgressOffset = progressOffset;
+}
+
 void CMtProgressMixer2::SetProgressOffset(UInt64 progressOffset)
 {
   CriticalSection.Enter();
-  InSizes[1] = OutSizes[1] = 0;
-  ProgressOffset = progressOffset;
+  SetProgressOffset_NoLock(progressOffset);
   CriticalSection.Leave();
 }
 
@@ -375,6 +391,7 @@ static HRESULT UpdateItemOldData(
     item.Comment = ui.Comment;
     item.Name = ui.Name;
     item.SetUtf8(ui.IsUtf8);
+    // item.SetFlag_AltStream(ui.IsAltStream);
     item.Time = ui.Time;
     item.Ntfs_MTime = ui.Ntfs_MTime;
     item.Ntfs_ATime = ui.Ntfs_ATime;
@@ -412,7 +429,7 @@ static HRESULT UpdateItemOldData(
 static void WriteDirHeader(COutArchive &archive, const CCompressionMethodMode *options,
     const CUpdateItem &ui, CItemOut &item)
 {
-  SetFileHeader(*options, ui, item);
+  SetFileHeader(*options, ui, false, item);
   archive.WriteLocalHeader(item);
 }
 
@@ -471,7 +488,7 @@ static HRESULT Update2St(
     CInArchive *inArchive,
     const CObjectVector<CItemEx> &inputItems,
     CObjectVector<CUpdateItem> &updateItems,
-    const CCompressionMethodMode *options,
+    const CCompressionMethodMode *options, bool outSeqMode,
     const CByteBuffer *comment,
     IArchiveUpdateCallback *updateCallback,
     UInt64 &totalComplexity,
@@ -527,28 +544,28 @@ static HRESULT Update2St(
         if (!fileInStream)
           return E_INVALIDARG;
 
-        bool seqMode;
+        bool inSeqMode = false;
+        if (!inSeqMode)
         {
           CMyComPtr<IInStream> inStream2;
           fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
-          seqMode = (inStream2 == NULL);
+          inSeqMode = (inStream2 == NULL);
         }
         // seqMode = true; // to test seqMode
 
         UpdatePropsFromStream(ui, fileInStream, updateCallback, totalComplexity);
-        SetFileHeader(*options, ui, item);
-
-        item.SetDescriptorMode(seqMode);
-
-        // file Size can be 64-bit !!!
 
         CCompressingResult compressingResult;
         
         RINOK(compressor.Set_Pre_CompressionResult(
-            seqMode,
+            inSeqMode, outSeqMode,
             ui.Size,
             compressingResult));
 
+        SetFileHeader(*options, ui, compressingResult.DescriptorMode, item);
+
+        // file Size can be 64-bit !!!
+
         SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
 
         archive.WriteLocalHeader(item);
@@ -559,17 +576,12 @@ static HRESULT Update2St(
         RINOK(compressor.Compress(
             EXTERNAL_CODECS_LOC_VARS
             fileInStream, outStream,
-            seqMode, ui.Time,
+            inSeqMode, outSeqMode,
+            ui.Time, ui.Size,
             progress, compressingResult));
         
-        if (compressingResult.FileTimeWasUsed)
-        {
-          /*
-          if (!item.HasDescriptor())
-            return E_FAIL;
-          */
-          item.SetDescriptorMode(true);
-        }
+        if (item.HasDescriptor() != compressingResult.DescriptorMode)
+          return E_FAIL;
 
         SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
 
@@ -598,8 +610,11 @@ static HRESULT Update2St(
   lps->InSize = unpackSizeTotal;
   lps->OutSize = packSizeTotal;
   RINOK(lps->SetCur());
+
   archive.WriteCentralDir(items, comment);
-  return S_OK;
+
+  lps->ProgressOffset += kCentralHeaderSize * updateItems.Size() + 1;
+  return lps->SetCur();
 }
 
 
@@ -609,7 +624,7 @@ static HRESULT Update2(
     CInArchive *inArchive,
     const CObjectVector<CItemEx> &inputItems,
     CObjectVector<CUpdateItem> &updateItems,
-    const CCompressionMethodMode &options,
+    const CCompressionMethodMode &options, bool outSeqMode,
     const CByteBuffer *comment,
     IArchiveUpdateCallback *updateCallback)
 {
@@ -743,32 +758,38 @@ static HRESULT Update2(
         const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
         const UInt32 blockSize = oneMethodMain->Get_BZip2_BlockSize();
         const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1;
-        numBZip2Threads = 32;
+        numBZip2Threads = 64;
         if (numBZip2Threads > averageNumberOfBlocks)
           numBZip2Threads = (UInt32)averageNumberOfBlocks;
+        if (numBZip2Threads > numThreads)
+          numBZip2Threads = numThreads;
         oneMethodMain->AddProp_NumThreads(numBZip2Threads);
       }
       numThreads /= numBZip2Threads;
     }
-   
-    if (method == NFileHeader::NCompressionMethod::kXz)
+    else if (method == NFileHeader::NCompressionMethod::kXz)
     {
-      bool fixedNumber;
-      UInt32 numLzma2Threads = oneMethodMain->Get_Lzma2_NumThreads(fixedNumber);
-      if (!fixedNumber)
+      UInt32 numLzmaThreads = 1;
+      int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads);
+      if (numXzThreads < 0)
       {
         const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
-        const UInt64 blockSize = oneMethodMain->Get_Lzma2_BlockSize();
-        const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1;
-        numLzma2Threads = 2;
-        if (numLzma2Threads > averageNumberOfBlocks)
-          numLzma2Threads = (UInt32)averageNumberOfBlocks;
-        oneMethodMain->AddProp_NumThreads(numLzma2Threads);
+        const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize();
+        UInt64 averageNumberOfBlocks = 1;
+        if (blockSize != (UInt64)(Int64)-1)
+          averageNumberOfBlocks = averageSize / blockSize + 1;
+        UInt32 t = 256;
+        if (t > averageNumberOfBlocks)
+          t = (UInt32)averageNumberOfBlocks;
+        t *= numLzmaThreads;
+        if (t > numThreads)
+          t = numThreads;
+        oneMethodMain->AddProp_NumThreads(t);
+        numXzThreads = t;
       }
-      numThreads /= numLzma2Threads;
+      numThreads /= (unsigned)numXzThreads;
     }
-
-    if (method == NFileHeader::NCompressionMethod::kLZMA)
+    else if (method == NFileHeader::NCompressionMethod::kLZMA)
     {
       // we suppose that default LZMA is 2 thread. So we don't change it
       UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads();
@@ -782,12 +803,14 @@ static HRESULT Update2(
       mtMode = false;
   }
 
+  // mtMode = true; // to test mtMode for seqMode
+
   if (!mtMode)
   #endif
     return Update2St(
         EXTERNAL_CODECS_LOC_VARS
         archive, inArchive,
-        inputItems, updateItems, &options2, comment, updateCallback, totalComplexity, opCallback);
+        inputItems, updateItems, &options2, outSeqMode, comment, updateCallback, totalComplexity, opCallback);
 
 
   #ifndef _7ZIP_ST
@@ -830,8 +853,10 @@ static HRESULT Update2(
       threadInfo.ProgressSpec = new CMtCompressProgress();
       threadInfo.Progress = threadInfo.ProgressSpec;
       threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, (int)i);
-      threadInfo.SeqMode = false; // fix it !
-      threadInfo.FileTime = 0; // fix it !
+      threadInfo.InSeqMode = false;
+      threadInfo.OutSeqMode = false;
+      threadInfo.FileTime = 0;
+      threadInfo.ExpectedDataSize = (UInt64)(Int64)-1;
       RINOK(threadInfo.CreateThread());
     }
   }
@@ -840,10 +865,15 @@ static HRESULT Update2(
   unsigned itemIndex = 0;
   int lastRealStreamItemIndex = -1;
 
+  
   while (itemIndex < updateItems.Size())
   {
     if (threadIndices.Size() < numThreads && mtItemIndex < updateItems.Size())
     {
+      // we start ahead the threads for compressing
+      // also we set refs.Refs[itemIndex].SeqMode that is used later
+      // don't move that code block
+      
       CUpdateItem &ui = updateItems[mtItemIndex++];
       if (!ui.NewData)
         continue;
@@ -869,6 +899,8 @@ static HRESULT Update2(
       
       CMyComPtr<ISequentialInStream> fileInStream;
       
+      CMemBlocks2 &memRef2 = refs.Refs[mtItemIndex - 1];
+
       {
         NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection);
         HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream);
@@ -876,9 +908,9 @@ static HRESULT Update2(
         {
           complexity += ui.Size;
           complexity += kLocalHeaderSize;
-          mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
+          mtProgressMixerSpec->Mixer2->SetProgressOffset_NoLock(complexity);
           RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK));
-          refs.Refs[mtItemIndex - 1].Skip = true;
+          memRef2.Skip = true;
           continue;
         }
         RINOK(res);
@@ -888,26 +920,46 @@ static HRESULT Update2(
         RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK));
       }
 
-      for (UInt32 k = 0; k < numThreads; k++)
+      UInt32 k;
+      for (k = 0; k < numThreads; k++)
+        if (threads.Threads[k].IsFree)
+          break;
+
+      if (k == numThreads)
+        return E_FAIL;
       {
-        CThreadInfo &threadInfo = threads.Threads[k];
-        if (threadInfo.IsFree)
         {
+          CThreadInfo &threadInfo = threads.Threads[k];
           threadInfo.IsFree = false;
           threadInfo.InStream = fileInStream;
 
+          bool inSeqMode = false;
+
+          if (!inSeqMode)
+          {
+            CMyComPtr<IInStream> inStream2;
+            fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
+            inSeqMode = (inStream2 == NULL);
+          }
+          memRef2.InSeqMode = inSeqMode;
+
           // !!!!! we must release ref before sending event
           // BUG was here in v4.43 and v4.44. It could change ref counter in two threads in same time
           fileInStream.Release();
 
           threadInfo.OutStreamSpec->Init();
           threadInfo.ProgressSpec->Reinit();
-          threadInfo.CompressEvent.Set();
+          
           threadInfo.UpdateIndex = mtItemIndex - 1;
-
+          threadInfo.InSeqMode = inSeqMode;
+          threadInfo.OutSeqMode = outSeqMode;
+          threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode
+          threadInfo.ExpectedDataSize = ui.Size;
+          
+          threadInfo.CompressEvent.Set();
+          
           compressingCompletedEvents.Add(threadInfo.CompressionCompletedEvent);
           threadIndices.Add(k);
-          break;
         }
       }
  
@@ -946,15 +998,17 @@ static HRESULT Update2(
       {
         CMemBlocks2 &memRef = refs.Refs[itemIndex];
         
-        if (memRef.Defined)
+        if (memRef.Finished)
         {
           if (lastRealStreamItemIndex < (int)itemIndex)
             lastRealStreamItemIndex = itemIndex;
 
-          SetFileHeader(options, ui, item);
+          SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item);
+
           // the BUG was fixed in 9.26:
           // SetItemInfoFromCompressingResult must be after SetFileHeader
           // to write correct Size.
+
           SetItemInfoFromCompressingResult(memRef.CompressingResult,
               options.IsRealAesMode(), options.AesKeyMode, item);
           archive.WriteLocalHeader(item);
@@ -967,16 +1021,25 @@ static HRESULT Update2(
         }
         else
         {
+          // current file was not finished
+
           if (lastRealStreamItemIndex < (int)itemIndex)
           {
+            // LocalHeader was not written for current itemIndex still
+
             lastRealStreamItemIndex = itemIndex;
-            SetFileHeader(options, ui, item);
+
+            // thread was started before for that item already, and memRef.SeqMode was set
 
             CCompressingResult compressingResult;
             RINOK(compressor.Set_Pre_CompressionResult(
-                false, // seqMode
+                memRef.InSeqMode, outSeqMode,
                 ui.Size,
                 compressingResult));
+
+            memRef.PreDescriptorMode = compressingResult.DescriptorMode;
+            SetFileHeader(options, ui, compressingResult.DescriptorMode, item);
+
             SetItemInfoFromCompressingResult(compressingResult, options.IsRealAesMode(), options.AesKeyMode, item);
 
             // file Size can be 64-bit !!!
@@ -1015,19 +1078,29 @@ static HRESULT Update2(
           
           if (t == 0)
           {
+            // if thread for current file was finished.
+            if (threadInfo.UpdateIndex != itemIndex)
+              return E_FAIL;
+
+            if (memRef.PreDescriptorMode != threadInfo.CompressingResult.DescriptorMode)
+              return E_FAIL;
+
             RINOK(threadInfo.OutStreamSpec->WriteToRealStream());
             threadInfo.OutStreamSpec->ReleaseOutStream();
-            SetFileHeader(options, ui, item);
+            SetFileHeader(options, ui, threadInfo.CompressingResult.DescriptorMode, item);
             SetItemInfoFromCompressingResult(threadInfo.CompressingResult,
                 options.IsRealAesMode(), options.AesKeyMode, item);
+
             archive.WriteLocalHeader_Replace(item);
           }
           else
           {
+            // it's not current file. So we must store information in array
             CMemBlocks2 &memRef2 = refs.Refs[threadInfo.UpdateIndex];
             threadInfo.OutStreamSpec->DetachData(memRef2);
             memRef2.CompressingResult = threadInfo.CompressingResult;
-            memRef2.Defined = true;
+            // memRef2.SeqMode = threadInfo.SeqMode; // it was set before
+            memRef2.Finished = true;
             continue;
           }
         }
@@ -1045,8 +1118,13 @@ static HRESULT Update2(
   }
   
   RINOK(mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL));
+
   archive.WriteCentralDir(items, comment);
-  return S_OK;
+  
+  complexity += kCentralHeaderSize * updateItems.Size() + 1;
+  mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
+  return mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL);
+
   #endif
 }
 
@@ -1060,6 +1138,7 @@ class CCacheOutStream:
   public CMyUnknownImp
 {
   CMyComPtr<IOutStream> _stream;
+  CMyComPtr<ISequentialOutStream> _seqStream;
   Byte *_cache;
   UInt64 _virtPos;
   UInt64 _virtSize;
@@ -1075,10 +1154,10 @@ class CCacheOutStream:
   }
   HRESULT FlushCache();
 public:
-  CCacheOutStream(): _cache(0) {}
+  CCacheOutStream(): _cache(NULL) {}
   ~CCacheOutStream();
   bool Allocate();
-  HRESULT Init(IOutStream *stream);
+  HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream);
   
   MY_UNKNOWN_IMP
 
@@ -1094,13 +1173,19 @@ bool CCacheOutStream::Allocate()
   return (_cache != NULL);
 }
 
-HRESULT CCacheOutStream::Init(IOutStream *stream)
+HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream)
 {
-  _virtPos = _phyPos = 0;
+  _virtPos = 0;
+  _phyPos = 0;
+  _virtSize = 0;
+  _seqStream = seqStream;
   _stream = stream;
-  RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos));
-  RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize));
-  RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos));
+  if (_stream)
+  {
+    RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos));
+    RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize));
+    RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos));
+  }
   _phyPos = _virtPos;
   _phySize = _virtSize;
   _cachedPos = 0;
@@ -1114,12 +1199,14 @@ HRESULT CCacheOutStream::MyWrite(size_t size)
   {
     if (_phyPos != _cachedPos)
     {
+      if (!_stream)
+        return E_FAIL;
       RINOK(_stream->Seek(_cachedPos, STREAM_SEEK_SET, &_phyPos));
     }
     size_t pos = (size_t)_cachedPos & kCacheMask;
     size_t curSize = MyMin(kCacheSize - pos, _cachedSize);
     curSize = MyMin(curSize, size);
-    RINOK(WriteStream(_stream, _cache + pos, curSize));
+    RINOK(WriteStream(_seqStream, _cache + pos, curSize));
     _phyPos += curSize;
     if (_phySize < _phyPos)
       _phySize = _phyPos;
@@ -1138,10 +1225,13 @@ HRESULT CCacheOutStream::FlushCache()
 CCacheOutStream::~CCacheOutStream()
 {
   FlushCache();
-  if (_virtSize != _phySize)
-    _stream->SetSize(_virtSize);
-  if (_virtPos != _phyPos)
-    _stream->Seek(_virtPos, STREAM_SEEK_SET, NULL);
+  if (_stream)
+  {
+    if (_virtSize != _phySize)
+      _stream->SetSize(_virtSize);
+    if (_virtPos != _phyPos)
+      _stream->Seek(_virtPos, STREAM_SEEK_SET, NULL);
+  }
   ::MidFree(_cache);
 }
 
@@ -1250,6 +1340,8 @@ STDMETHODIMP CCacheOutStream::SetSize(UInt64 newSize)
   _virtSize = newSize;
   if (newSize < _phySize)
   {
+    if (!_stream)
+      return E_NOTIMPL;
     RINOK(_stream->SetSize(newSize));
     _phySize = newSize;
   }
@@ -1281,11 +1373,14 @@ HRESULT Update(
 
 
   CMyComPtr<IOutStream> outStream;
+  bool outSeqMode;
   {
     CMyComPtr<IOutStream> outStreamReal;
     seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStreamReal);
     if (!outStreamReal)
-      return E_NOTIMPL;
+    {
+      // return E_NOTIMPL;
+    }
 
     if (inArchive)
     {
@@ -1293,7 +1388,7 @@ HRESULT Update(
       {
         IInStream *baseStream = inArchive->GetBaseStream();
         RINOK(baseStream->Seek(0, STREAM_SEEK_SET, NULL));
-        RINOK(NCompress::CopyStream_ExactSize(baseStream, outStreamReal, inArchive->ArcInfo.Base, NULL));
+        RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, inArchive->ArcInfo.Base, NULL));
       }
     }
 
@@ -1301,7 +1396,8 @@ HRESULT Update(
     outStream = cacheStream;
     if (!cacheStream->Allocate())
       return E_OUTOFMEMORY;
-    RINOK(cacheStream->Init(outStreamReal));
+    RINOK(cacheStream->Init(seqOutStream, outStreamReal));
+    outSeqMode = (outStreamReal == NULL);
   }
 
   COutArchive outArchive;
@@ -1323,7 +1419,7 @@ HRESULT Update(
       EXTERNAL_CODECS_LOC_VARS
       outArchive, inArchive,
       inputItems, updateItems,
-      compressionMethodMode,
+      compressionMethodMode, outSeqMode,
       inArchive ? &inArchive->ArcInfo.Comment : NULL,
       updateCallback);
 }

CPP/7zip/Archive/Zip/ZipUpdate.h

@@ -32,6 +32,7 @@ struct CUpdateItem
   bool IsDir;
   bool NtfsTimeIsDefined;
   bool IsUtf8;
+  // bool IsAltStream;
   int IndexInArc;
   int IndexInClient;
   UInt32 Attrib;
@@ -50,12 +51,19 @@ struct CUpdateItem
     IsDir = false;
     NtfsTimeIsDefined = false;
     IsUtf8 = false;
+    // IsAltStream = false;
     Size = 0;
     Name.Empty();
     Comment.Free();
   }
 
-  CUpdateItem(): NtfsTimeIsDefined(false), IsUtf8(false), Size(0) {}
+  CUpdateItem():
+    IsDir(false),
+    NtfsTimeIsDefined(false),
+    IsUtf8(false),
+    // IsAltStream(false),
+    Size(0)
+    {}
 };
 
 HRESULT Update(

CPP/7zip/Bundles/Alone/Alone.dsp

@@ -2632,6 +2632,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+
+!IF  "$(CFG)" == "Alone - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 Debug"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 ReleaseU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 DebugU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Lzma2Enc.c
 
 !IF  "$(CFG)" == "Alone - Win32 Release"
@@ -2744,6 +2772,34 @@ SOURCE=..\..\..\..\C\MtCoder.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+
+!IF  "$(CFG)" == "Alone - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 Debug"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 ReleaseU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 DebugU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Ppmd.h
 # End Source File
 # Begin Source File

CPP/7zip/Bundles/Alone/makefile

@@ -198,10 +198,12 @@ C_OBJS = \
   $O\LzFind.obj \
   $O\LzFindMt.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\Lzma2Enc.obj \
   $O\LzmaDec.obj \
   $O\LzmaEnc.obj \
   $O\MtCoder.obj \
+  $O\MtDec.obj \
   $O\Ppmd7.obj \
   $O\Ppmd7Dec.obj \
   $O\Ppmd7Enc.obj \
@@ -222,5 +224,6 @@ C_OBJS = \
 !include "../../Aes.mak"
 !include "../../Crc.mak"
 !include "../../Crc64.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Alone7z/Alone.dsp

@@ -1696,6 +1696,34 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+
+!IF  "$(CFG)" == "Alone - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 Debug"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 ReleaseU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 DebugU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Lzma2Enc.c
 
 !IF  "$(CFG)" == "Alone - Win32 Release"
@@ -1789,6 +1817,34 @@ SOURCE=..\..\..\..\C\MtCoder.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+
+!IF  "$(CFG)" == "Alone - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 Debug"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 ReleaseU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "Alone - Win32 DebugU"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Threads.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File

CPP/7zip/Bundles/Alone7z/makefile

@@ -133,10 +133,12 @@ C_OBJS = \
   $O\LzFind.obj \
   $O\LzFindMt.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\Lzma2Enc.obj \
   $O\LzmaDec.obj \
   $O\LzmaEnc.obj \
   $O\MtCoder.obj \
+  $O\MtDec.obj \
   $O\Sha256.obj \
   $O\Sort.obj \
   $O\Threads.obj \
@@ -150,5 +152,6 @@ C_OBJS = \
 !include "../../Aes.mak"
 !include "../../Crc.mak"
 !include "../../Crc64.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Fm/FM.dsp

@@ -1004,6 +1004,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Lzma2Enc.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File
@@ -1040,6 +1049,15 @@ SOURCE=..\..\..\..\C\MtCoder.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Sha256.c
 
 !IF  "$(CFG)" == "FM - Win32 Release"

CPP/7zip/Bundles/Format7z/makefile

@@ -123,10 +123,12 @@ C_OBJS = \
   $O\LzFind.obj \
   $O\LzFindMt.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\Lzma2Enc.obj \
   $O\LzmaDec.obj \
   $O\LzmaEnc.obj \
   $O\MtCoder.obj \
+  $O\MtDec.obj \
   $O\Ppmd7.obj \
   $O\Ppmd7Dec.obj \
   $O\Ppmd7Enc.obj \
@@ -136,5 +138,6 @@ C_OBJS = \
 
 !include "../../Aes.mak"
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zExtract/makefile

@@ -42,6 +42,7 @@ AR_OBJS = \
 
 AR_COMMON_OBJS = \
   $O\CoderMixer2.obj \
+  $O\HandlerOut.obj \
   $O\ItemNameUtils.obj \
   $O\OutStreamWithCRC.obj \
   $O\ParseProperties.obj \
@@ -98,7 +99,9 @@ C_OBJS = \
   $O\CpuArch.obj \
   $O\Delta.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\LzmaDec.obj \
+  $O\MtDec.obj \
   $O\Ppmd7.obj \
   $O\Ppmd7Dec.obj \
   $O\Sha256.obj \
@@ -106,5 +109,6 @@ C_OBJS = \
 
 !include "../../Aes.mak"
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zExtractR/makefile

@@ -85,9 +85,12 @@ C_OBJS = \
   $O\CpuArch.obj \
   $O\Delta.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\LzmaDec.obj \
+  $O\MtDec.obj \
   $O\Threads.obj \
 
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/Format7zF/Arc.mak

@@ -198,6 +198,7 @@ COMPRESS_OBJS = \
   $O\DeflateRegister.obj \
   $O\DeltaFilter.obj \
   $O\ImplodeDecoder.obj \
+  $O\LzfseDecoder.obj \
   $O\LzhDecoder.obj \
   $O\Lzma2Decoder.obj \
   $O\Lzma2Encoder.obj \
@@ -262,10 +263,12 @@ C_OBJS = \
   $O\LzFind.obj \
   $O\LzFindMt.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\Lzma2Enc.obj \
   $O\LzmaDec.obj \
   $O\LzmaEnc.obj \
   $O\MtCoder.obj \
+  $O\MtDec.obj \
   $O\Ppmd7.obj \
   $O\Ppmd7Dec.obj \
   $O\Ppmd7Enc.obj \
@@ -284,3 +287,4 @@ C_OBJS = \
 !include "../../Aes.mak"
 !include "../../Crc.mak"
 !include "../../Crc64.mak"
+!include "../../LzmaDec.mak"

CPP/7zip/Bundles/Format7zF/Format7z.dsp

@@ -961,6 +961,24 @@ SOURCE=..\..\Compress\HuffmanDecoder.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\Compress\LzfseDecoder.cpp
+
+!IF  "$(CFG)" == "7z - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "7z - Win32 Debug"
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\Compress\LzfseDecoder.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\Compress\LzhDecoder.cpp
 # End Source File
 # Begin Source File
@@ -1787,6 +1805,26 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+
+!IF  "$(CFG)" == "7z - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "7z - Win32 Debug"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Lzma2Enc.c
 
 !IF  "$(CFG)" == "7z - Win32 Release"
@@ -1856,6 +1894,26 @@ SOURCE=..\..\..\..\C\MtCoder.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+
+!IF  "$(CFG)" == "7z - Win32 Release"
+
+# ADD CPP /O2
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ELSEIF  "$(CFG)" == "7z - Win32 Debug"
+
+# SUBTRACT CPP /YX /Yc /Yu
+
+!ENDIF 
+
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Ppmd.h
 # End Source File
 # Begin Source File

CPP/7zip/Bundles/Format7zR/makefile

@@ -102,12 +102,15 @@ C_OBJS = \
   $O\LzFind.obj \
   $O\LzFindMt.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\Lzma2Enc.obj \
   $O\LzmaDec.obj \
   $O\LzmaEnc.obj \
   $O\MtCoder.obj \
+  $O\MtDec.obj \
   $O\Threads.obj \
 
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/SFXCon/SFXCon.dsp

@@ -117,6 +117,14 @@ SOURCE=..\..\Archive\Common\CoderMixer2.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\Archive\Common\HandlerOut.cpp
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\Archive\Common\HandlerOut.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\Archive\Common\ItemNameUtils.cpp
 # End Source File
 # Begin Source File
@@ -293,6 +301,10 @@ SOURCE=..\..\Compress\Lzma2Decoder.cpp
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\Compress\Lzma2Decoder.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\Compress\Lzma2Register.cpp
 # End Source File
 # Begin Source File
@@ -411,6 +423,14 @@ SOURCE=..\..\..\Windows\Synchronization.cpp
 
 SOURCE=..\..\..\Windows\Synchronization.h
 # End Source File
+# Begin Source File
+
+SOURCE=..\..\..\Windows\System.cpp
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\Windows\System.h
+# End Source File
 # End Group
 # Begin Group "Common"
 
@@ -822,6 +842,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\LzmaDec.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File
@@ -831,6 +860,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Ppmd7.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File

CPP/7zip/Bundles/SFXCon/makefile

@@ -41,6 +41,7 @@ WIN_OBJS = \
   $O\PropVariant.obj \
   $O\PropVariantConv.obj \
   $O\Synchronization.obj \
+  $O\System.obj \
 
 7ZIP_COMMON_OBJS = \
   $O\CreateCoder.obj \
@@ -117,7 +118,9 @@ C_OBJS = \
   $O\Delta.obj \
   $O\DllSecur.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\LzmaDec.obj \
+  $O\MtDec.obj \
   $O\Ppmd7.obj \
   $O\Ppmd7Dec.obj \
   $O\Sha256.obj \
@@ -125,5 +128,6 @@ C_OBJS = \
 
 !include "../../Aes.mak"
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/SFXSetup/SFXSetup.dsp

@@ -449,6 +449,14 @@ SOURCE=..\..\..\Windows\Synchronization.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\Windows\System.cpp
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\Windows\System.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\Windows\Window.cpp
 # End Source File
 # Begin Source File
@@ -469,6 +477,14 @@ SOURCE=..\..\Common\CreateCoder.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\Common\CWrappers.cpp
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\Common\CWrappers.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\Common\FileStreams.cpp
 # End Source File
 # Begin Source File
@@ -724,6 +740,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\LzmaDec.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File
@@ -733,6 +758,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Threads.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File

CPP/7zip/Bundles/SFXSetup/makefile

@@ -33,6 +33,7 @@ WIN_OBJS = \
   $O\PropVariant.obj \
   $O\ResourceString.obj \
   $O\Synchronization.obj \
+  $O\System.obj \
   $O\Window.obj \
 
 WIN_CTRL_OBJS = \
@@ -40,6 +41,7 @@ WIN_CTRL_OBJS = \
 
 7ZIP_COMMON_OBJS = \
   $O\CreateCoder.obj \
+  $O\CWrappers.obj \
   $O\FileStreams.obj \
   $O\InBuffer.obj \
   $O\FilterCoder.obj \
@@ -102,9 +104,12 @@ C_OBJS = \
   $O\Delta.obj \
   $O\DllSecur.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\LzmaDec.obj \
+  $O\MtDec.obj \
   $O\Threads.obj \
 
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Bundles/SFXWin/SFXWin.dsp

@@ -633,6 +633,14 @@ SOURCE=..\..\..\Windows\Synchronization.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\Windows\System.cpp
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\Windows\System.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\Windows\Window.cpp
 # End Source File
 # Begin Source File
@@ -906,6 +914,15 @@ SOURCE=..\..\..\..\C\Lzma2Dec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\Lzma2DecMt.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\Lzma2DecMt.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\LzmaDec.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File
@@ -915,6 +932,15 @@ SOURCE=..\..\..\..\C\LzmaDec.h
 # End Source File
 # Begin Source File
 
+SOURCE=..\..\..\..\C\MtDec.c
+# SUBTRACT CPP /YX /Yc /Yu
+# End Source File
+# Begin Source File
+
+SOURCE=..\..\..\..\C\MtDec.h
+# End Source File
+# Begin Source File
+
 SOURCE=..\..\..\..\C\Ppmd7.c
 # SUBTRACT CPP /YX /Yc /Yu
 # End Source File

CPP/7zip/Bundles/SFXWin/makefile

@@ -41,6 +41,7 @@ WIN_OBJS = \
   $O\ResourceString.obj \
   $O\Shell.obj \
   $O\Synchronization.obj \
+  $O\System.obj \
   $O\Window.obj \
 
 WIN_CTRL_OBJS = \
@@ -135,7 +136,9 @@ C_OBJS = \
   $O\Delta.obj \
   $O\DllSecur.obj \
   $O\Lzma2Dec.obj \
+  $O\Lzma2DecMt.obj \
   $O\LzmaDec.obj \
+  $O\MtDec.obj \
   $O\Ppmd7.obj \
   $O\Ppmd7Dec.obj \
   $O\Sha256.obj \
@@ -143,5 +146,6 @@ C_OBJS = \
 
 !include "../../Aes.mak"
 !include "../../Crc.mak"
+!include "../../LzmaDec.mak"
 
 !include "../../7zip.mak"

CPP/7zip/Common/CWrappers.cpp

@@ -28,13 +28,26 @@ HRESULT SResToHRESULT(SRes res) throw()
   switch (res)
   {
     case SZ_OK: return S_OK;
+    
+    case SZ_ERROR_DATA:
+    case SZ_ERROR_CRC:
+    case SZ_ERROR_INPUT_EOF:
+      return S_FALSE;
+    
     case SZ_ERROR_MEM: return E_OUTOFMEMORY;
     case SZ_ERROR_PARAM: return E_INVALIDARG;
     case SZ_ERROR_PROGRESS: return E_ABORT;
-    case SZ_ERROR_DATA: return S_FALSE;
     case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL;
-    // case SZ_ERROR_READ: return E_NOTIMPL;
+    // case SZ_ERROR_OUTPUT_EOF:
+    // case SZ_ERROR_READ:
+    // case SZ_ERROR_WRITE:
+    // case SZ_ERROR_THREAD:
+    // case SZ_ERROR_ARCHIVE:
+    // case SZ_ERROR_NO_ARCHIVE:
+    // return E_FAIL;
   }
+  if (res < 0)
+    return res;
   return E_FAIL;
 }
 

CPP/7zip/Common/CreateCoder.cpp

@@ -148,20 +148,23 @@ HRESULT CExternalCodecs::Load()
 #endif
 
 
-bool FindMethod(
+int FindMethod_Index(
     DECL_EXTERNAL_CODECS_LOC_VARS
     const AString &name,
-    CMethodId &methodId, UInt32 &numStreams)
+    bool encode,
+    CMethodId &methodId,
+    UInt32 &numStreams)
 {
   unsigned i;
   for (i = 0; i < g_NumCodecs; i++)
   {
     const CCodecInfo &codec = *g_Codecs[i];
-    if (StringsAreEqualNoCase_Ascii(name, codec.Name))
+    if ((encode ? codec.CreateEncoder : codec.CreateDecoder)
+        && StringsAreEqualNoCase_Ascii(name, codec.Name))
     {
       methodId = codec.Id;
       numStreams = codec.NumStreams;
-      return true;
+      return i;
     }
   }
   
@@ -173,19 +176,51 @@ bool FindMethod(
     for (i = 0; i < __externalCodecs->Codecs.Size(); i++)
     {
       const CCodecInfoEx &codec = __externalCodecs->Codecs[i];
-      if (StringsAreEqualNoCase_Ascii(name, codec.Name))
+      if ((encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned)
+          && StringsAreEqualNoCase_Ascii(name, codec.Name))
       {
         methodId = codec.Id;
         numStreams = codec.NumStreams;
-        return true;
+        return g_NumCodecs + i;
       }
     }
   
   #endif
   
-  return false;
+  return -1;
 }
 
+
+static int FindMethod_Index(
+    DECL_EXTERNAL_CODECS_LOC_VARS
+    CMethodId methodId, bool encode)
+{
+  unsigned i;
+  for (i = 0; i < g_NumCodecs; i++)
+  {
+    const CCodecInfo &codec = *g_Codecs[i];
+    if (codec.Id == methodId && (encode ? codec.CreateEncoder : codec.CreateDecoder))
+      return i;
+  }
+  
+  #ifdef EXTERNAL_CODECS
+  
+  CHECK_GLOBAL_CODECS
+
+  if (__externalCodecs)
+    for (i = 0; i < __externalCodecs->Codecs.Size(); i++)
+    {
+      const CCodecInfoEx &codec = __externalCodecs->Codecs[i];
+      if (codec.Id == methodId && (encode ? codec.EncoderIsAssigned : codec.DecoderIsAssigned))
+        return g_NumCodecs + i;
+    }
+  
+  #endif
+  
+  return -1;
+}
+
+
 bool FindMethod(
     DECL_EXTERNAL_CODECS_LOC_VARS
     CMethodId methodId,
@@ -280,9 +315,11 @@ void GetHashMethods(
   #endif
 }
 
-HRESULT CreateCoder(
+
+
+HRESULT CreateCoder_Index(
     DECL_EXTERNAL_CODECS_LOC_VARS
-    CMethodId methodId, bool encode,
+    unsigned i, bool encode,
     CMyComPtr<ICompressFilter> &filter,
     CCreatedCoder &cod)
 {
@@ -290,11 +327,10 @@ HRESULT CreateCoder(
   cod.IsFilter = false;
   cod.NumStreams = 1;
 
-  unsigned i;
-  for (i = 0; i < g_NumCodecs; i++)
+  if (i < g_NumCodecs)
   {
     const CCodecInfo &codec = *g_Codecs[i];
-    if (codec.Id == methodId)
+    // if (codec.Id == methodId)
     {
       if (encode)
       {
@@ -325,11 +361,12 @@ HRESULT CreateCoder(
   
   if (__externalCodecs)
   {
+    i -= g_NumCodecs;
     cod.IsExternal = true;
-    for (i = 0; i < __externalCodecs->Codecs.Size(); i++)
+    if (i < __externalCodecs->Codecs.Size())
     {
       const CCodecInfoEx &codec = __externalCodecs->Codecs[i];
-      if (codec.Id == methodId)
+      // if (codec.Id == methodId)
       {
         if (encode)
         {
@@ -371,13 +408,50 @@ HRESULT CreateCoder(
   return S_OK;
 }
 
-HRESULT CreateCoder(
+
+HRESULT CreateCoder_Index(
+    DECL_EXTERNAL_CODECS_LOC_VARS
+    unsigned index, bool encode,
+    CCreatedCoder &cod)
+{
+  CMyComPtr<ICompressFilter> filter;
+  HRESULT res = CreateCoder_Index(
+      EXTERNAL_CODECS_LOC_VARS
+      index, encode,
+      filter, cod);
+  
+  if (filter)
+  {
+    cod.IsFilter = true;
+    CFilterCoder *coderSpec = new CFilterCoder(encode);
+    cod.Coder = coderSpec;
+    coderSpec->Filter = filter;
+  }
+  
+  return res;
+}
+
+
+HRESULT CreateCoder_Id(
+    DECL_EXTERNAL_CODECS_LOC_VARS
+    CMethodId methodId, bool encode,
+    CMyComPtr<ICompressFilter> &filter,
+    CCreatedCoder &cod)
+{
+  int index = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS methodId, encode);
+  if (index < 0)
+    return S_OK;
+  return CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS index, encode, filter, cod);
+}
+
+
+HRESULT CreateCoder_Id(
     DECL_EXTERNAL_CODECS_LOC_VARS
     CMethodId methodId, bool encode,
     CCreatedCoder &cod)
 {
   CMyComPtr<ICompressFilter> filter;
-  HRESULT res = CreateCoder(
+  HRESULT res = CreateCoder_Id(
       EXTERNAL_CODECS_LOC_VARS
       methodId, encode,
       filter, cod);
@@ -393,13 +467,14 @@ HRESULT CreateCoder(
   return res;
 }
 
-HRESULT CreateCoder(
+
+HRESULT CreateCoder_Id(
     DECL_EXTERNAL_CODECS_LOC_VARS
     CMethodId methodId, bool encode,
     CMyComPtr<ICompressCoder> &coder)
 {
   CCreatedCoder cod;
-  HRESULT res = CreateCoder(
+  HRESULT res = CreateCoder_Id(
       EXTERNAL_CODECS_LOC_VARS
       methodId, encode,
       cod);
@@ -413,7 +488,7 @@ HRESULT CreateFilter(
     CMyComPtr<ICompressFilter> &filter)
 {
   CCreatedCoder cod;
-  return CreateCoder(
+  return CreateCoder_Id(
       EXTERNAL_CODECS_LOC_VARS
       methodId, encode,
       filter, cod);

CPP/7zip/Common/CreateCoder.h

@@ -116,13 +116,12 @@ extern CExternalCodecs g_ExternalCodecs;
 
 #endif
 
-
-
-
-bool FindMethod(
+int FindMethod_Index(
     DECL_EXTERNAL_CODECS_LOC_VARS
     const AString &name,
-    CMethodId &methodId, UInt32 &numStreams);
+    bool encode,
+    CMethodId &methodId,
+    UInt32 &numStreams);
 
 bool FindMethod(
     DECL_EXTERNAL_CODECS_LOC_VARS
@@ -152,18 +151,29 @@ struct CCreatedCoder
 };
 
 
-HRESULT CreateCoder(
+HRESULT CreateCoder_Index(
+    DECL_EXTERNAL_CODECS_LOC_VARS
+    unsigned codecIndex, bool encode,
+    CMyComPtr<ICompressFilter> &filter,
+    CCreatedCoder &cod);
+
+HRESULT CreateCoder_Index(
+    DECL_EXTERNAL_CODECS_LOC_VARS
+    unsigned index, bool encode,
+    CCreatedCoder &cod);
+
+HRESULT CreateCoder_Id(
     DECL_EXTERNAL_CODECS_LOC_VARS
     CMethodId methodId, bool encode,
     CMyComPtr<ICompressFilter> &filter,
     CCreatedCoder &cod);
 
-HRESULT CreateCoder(
+HRESULT CreateCoder_Id(
     DECL_EXTERNAL_CODECS_LOC_VARS
     CMethodId methodId, bool encode,
     CCreatedCoder &cod);
 
-HRESULT CreateCoder(
+HRESULT CreateCoder_Id(
     DECL_EXTERNAL_CODECS_LOC_VARS
     CMethodId methodId, bool encode,
     CMyComPtr<ICompressCoder> &coder);

CPP/7zip/Common/InBuffer.cpp

@@ -97,6 +97,33 @@ Byte CInBufferBase::ReadByte_FromNewBlock()
 
 size_t CInBufferBase::ReadBytes(Byte *buf, size_t size)
 {
+  size_t num = 0;
+  for (;;)
+  {
+    const size_t rem = _bufLim - _buf;
+    if (size <= rem)
+    {
+      if (size != 0)
+      {
+        memcpy(buf, _buf, size);
+        _buf += size;
+        num += size;
+      }
+      return num;
+    }
+    if (rem != 0)
+    {
+      memcpy(buf, _buf, rem);
+      _buf += rem;
+      buf += rem;
+      num += rem;
+      size -= rem;
+    }
+    if (!ReadBlock())
+      return num;
+  }
+
+  /*
   if ((size_t)(_bufLim - _buf) >= size)
   {
     const Byte *src = _buf;
@@ -113,6 +140,7 @@ size_t CInBufferBase::ReadBytes(Byte *buf, size_t size)
     buf[i] = *_buf++;
   }
   return size;
+  */
 }
 
 size_t CInBufferBase::Skip(size_t size)