xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-17 14:11:53 -06:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: xevion:17.00
Branches Tags
xevion:e7z-sf-without-zstd xevion:e7z-sf-zstd xevion:16.04-easy-orig-jameshoo xevion:Easy7Zip_1900 xevion:Easy7Zip
xevion:shunf4-fake-release xevion:18.05 xevion:18.03 xevion:18.01 xevion:18.00 xevion:17.01 xevion:17.00 xevion:16.04 xevion:16.03 xevion:16.02 xevion:16.00 xevion:16.01 xevion:15.13 xevion:15.12 xevion:15.14 xevion:15.10 xevion:15.11 xevion:15.08 xevion:15.09 xevion:15.06 xevion:15.07 xevion:15.05 xevion:9.35 xevion:9.36 xevion:9.38 xevion:9.34 xevion:9.22 xevion:9.20 xevion:9.21 xevion:9.17 xevion:9.18 xevion:9.19 xevion:9.16 xevion:9.14 xevion:9.13 xevion:9.15 xevion:9.11 xevion:9.12 xevion:4.64 xevion:4.65 xevion:4.63 xevion:4.62 xevion:4.57 xevion:4.42 xevion:4.31 xevion:4.32 xevion:4.23 xevion:3.13 xevion:4.20
...
compare: xevion:18.01
Branches Tags
xevion:e7z-sf-without-zstd xevion:e7z-sf-zstd xevion:16.04-easy-orig-jameshoo xevion:Easy7Zip_1900 xevion:Easy7Zip
xevion:shunf4-fake-release xevion:18.05 xevion:18.03 xevion:18.01 xevion:18.00 xevion:17.01 xevion:17.00 xevion:16.04 xevion:16.03 xevion:16.02 xevion:16.00 xevion:16.01 xevion:15.13 xevion:15.12 xevion:15.14 xevion:15.10 xevion:15.11 xevion:15.08 xevion:15.09 xevion:15.06 xevion:15.07 xevion:15.05 xevion:9.35 xevion:9.36 xevion:9.38 xevion:9.34 xevion:9.22 xevion:9.20 xevion:9.21 xevion:9.17 xevion:9.18 xevion:9.19 xevion:9.16 xevion:9.14 xevion:9.13 xevion:9.15 xevion:9.11 xevion:9.12 xevion:4.64 xevion:4.65 xevion:4.63 xevion:4.62 xevion:4.57 xevion:4.42 xevion:4.31 xevion:4.32 xevion:4.23 xevion:3.13 xevion:4.20

3 Commits
17.00 ... 18.01

Author SHA1 Message Date
Igor Pavlov
866a06f5a0 18.01 2018-01-30 00:35:06 +00:00
Igor Pavlov
da28077952 18.00 2018-01-11 22:16:32 +01:00
Igor Pavlov
b5dc853b24 17.01 2017-08-29 20:49:43 +01:00
145 changed files with 6949 additions and 2154 deletions
Expand all files Collapse all files

8
C/7zCrc.c
View File

@@ -1,5 +1,5 @@
/* 7zCrc.c -- CRC32 init
2017-04-03 : Igor Pavlov : Public domain */
2017-06-06 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -86,8 +86,8 @@ void MY_FAST_CALL CrcGenerateTable()
#ifdef MY_CPU_X86_OR_AMD64
if (!CPU_Is_InOrder())
g_CrcUpdate = CrcUpdateT8;
#endif
g_CrcUpdate = CrcUpdateT8;
#endif
#else
@@ -101,7 +101,7 @@ void MY_FAST_CALL CrcGenerateTable()
g_CrcUpdate = CrcUpdateT4;
#if CRC_NUM_TABLES >= 8
g_CrcUpdateT8 = CrcUpdateT8;
// g_CrcUpdate = CrcUpdateT8;
g_CrcUpdate = CrcUpdateT8;
#endif
}
else if (p[0] != 1 || p[1] != 2)
@@ -118,7 +118,7 @@ void MY_FAST_CALL CrcGenerateTable()
g_CrcUpdate = CrcUpdateT1_BeT4;
#if CRC_NUM_TABLES >= 8
g_CrcUpdateT8 = CrcUpdateT1_BeT8;
// g_CrcUpdate = CrcUpdateT1_BeT8;
g_CrcUpdate = CrcUpdateT1_BeT8;
#endif
}
}

12
C/7zTypes.h
View File

@@ -1,5 +1,5 @@
/* 7zTypes.h -- Basic types
2017-04-03 : Igor Pavlov : Public domain */
2017-07-17 : Igor Pavlov : Public domain */
#ifndef __7Z_TYPES_H
#define __7Z_TYPES_H
@@ -42,13 +42,23 @@ EXTERN_C_BEGIN
typedef int SRes;
#ifdef _WIN32
/* typedef DWORD WRes; */
typedef unsigned WRes;
#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x)
#else
typedef int WRes;
#define MY__FACILITY_WIN32 7
#define MY__FACILITY__WRes MY__FACILITY_WIN32
#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000)))
#endif
#ifndef RINOK
#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
#endif

10
C/7zVersion.h
View File

@@ -1,7 +1,7 @@
#define MY_VER_MAJOR 17
#define MY_VER_MINOR 00
#define MY_VER_MAJOR 18
#define MY_VER_MINOR 01
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "17.00 beta"
#define MY_VERSION_NUMBERS "18.01"
#define MY_VERSION MY_VERSION_NUMBERS
#ifdef MY_CPU_NAME
@@ -10,12 +10,12 @@
#define MY_VERSION_CPU MY_VERSION
#endif
#define MY_DATE "2017-04-29"
#define MY_DATE "2018-01-28"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov"
#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2017 Igor Pavlov"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov"
#ifdef USE_COPYRIGHT_CR
#define MY_COPYRIGHT MY_COPYRIGHT_CR

4
C/AesOpt.c
View File

@@ -1,12 +1,12 @@
/* AesOpt.c -- Intel's AES
2013-11-12 : Igor Pavlov : Public domain */
2017-06-08 : Igor Pavlov : Public domain */
#include "Precomp.h"
#include "CpuArch.h"
#ifdef MY_CPU_X86_OR_AMD64
#if _MSC_VER >= 1500
#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729)
#define USE_INTEL_AES
#endif
#endif

49
C/Alloc.c
View File

@@ -1,5 +1,5 @@
/* Alloc.c -- Memory allocation functions
2017-04-03 : Igor Pavlov : Public domain */
2017-06-15 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -23,11 +23,11 @@ int g_allocCountBig = 0;
void *MyAlloc(size_t size)
{
if (size == 0)
return 0;
return NULL;
#ifdef _SZ_ALLOC_DEBUG
{
void *p = malloc(size);
fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p);
fprintf(stderr, "\nAlloc %10u bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p);
return p;
}
#else
@@ -38,7 +38,7 @@ void *MyAlloc(size_t size)
void MyFree(void *address)
{
#ifdef _SZ_ALLOC_DEBUG
if (address != 0)
if (address)
fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address);
#endif
free(address);
@@ -49,20 +49,20 @@ void MyFree(void *address)
void *MidAlloc(size_t size)
{
if (size == 0)
return 0;
return NULL;
#ifdef _SZ_ALLOC_DEBUG
fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++);
#endif
return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
}
void MidFree(void *address)
{
#ifdef _SZ_ALLOC_DEBUG
if (address != 0)
if (address)
fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid);
#endif
if (address == 0)
if (!address)
return;
VirtualFree(address, 0, MEM_RELEASE);
}
@@ -79,10 +79,10 @@ typedef SIZE_T (WINAPI *GetLargePageMinimumP)();
void SetLargePageSize()
{
#ifdef _7ZIP_LARGE_PAGES
SIZE_T size = 0;
SIZE_T size;
GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP)
GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum");
if (largePageMinimum == 0)
if (!largePageMinimum)
return;
size = largePageMinimum();
if (size == 0 || (size & (size - 1)) != 0)
@@ -95,31 +95,40 @@ void SetLargePageSize()
void *BigAlloc(size_t size)
{
if (size == 0)
return 0;
return NULL;
#ifdef _SZ_ALLOC_DEBUG
fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++);
fprintf(stderr, "\nAlloc_Big %10u bytes; count = %10d", size, g_allocCountBig++);
#endif
#ifdef _7ZIP_LARGE_PAGES
if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18))
{
void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)),
MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
if (res != 0)
return res;
SIZE_T ps = g_LargePageSize;
if (ps != 0 && ps <= (1 << 30) && size > (ps / 2))
{
size_t size2;
ps--;
size2 = (size + ps) & ~ps;
if (size2 >= size)
{
void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
if (res)
return res;
}
}
}
#endif
return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
}
void BigFree(void *address)
{
#ifdef _SZ_ALLOC_DEBUG
if (address != 0)
if (address)
fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig);
#endif
if (address == 0)
if (!address)
return;
VirtualFree(address, 0, MEM_RELEASE);
}

22
C/CpuArch.h
View File

@@ -1,5 +1,5 @@
/* CpuArch.h -- CPU specific code
2017-04-03 : Igor Pavlov : Public domain */
2017-06-30 : Igor Pavlov : Public domain */
#ifndef __CPU_ARCH_H
#define __CPU_ARCH_H
@@ -190,8 +190,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#ifdef MY_CPU_LE
#if defined(MY_CPU_X86_OR_AMD64) \
|| defined(MY_CPU_ARM64) \
|| defined(__ARM_FEATURE_UNALIGNED) \
|| defined(__AARCH64EL__)
|| defined(__ARM_FEATURE_UNALIGNED)
#define MY_CPU_LE_UNALIGN
#endif
#endif
@@ -237,6 +236,11 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#endif
#ifdef __has_builtin
#define MY__has_builtin(x) __has_builtin(x)
#else
#define MY__has_builtin(x) 0
#endif
#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300)
@@ -244,15 +248,21 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#include <stdlib.h>
#pragma intrinsic(_byteswap_ushort)
#pragma intrinsic(_byteswap_ulong)
#pragma intrinsic(_byteswap_uint64)
/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */
#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p))
#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p))
#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v)
#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
#elif defined(MY_CPU_LE_UNALIGN) && ( \
(defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \
|| (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) )
/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */
#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p))
#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p))
@@ -277,10 +287,14 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#endif
#ifndef GetBe16
#define GetBe16(p) ( (UInt16) ( \
((UInt16)((const Byte *)(p))[0] << 8) | \
((const Byte *)(p))[1] ))
#endif
#ifdef MY_CPU_X86_OR_AMD64

47
C/LzFind.c
View File

@@ -1,5 +1,5 @@
/* LzFind.c -- Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2017-06-10 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -142,6 +142,7 @@ void MatchFinder_Construct(CMatchFinder *p)
p->bufferBase = NULL;
p->directInput = 0;
p->hash = NULL;
p->expectedDataSize = (UInt64)(Int64)-1;
MatchFinder_SetDefaultSettings(p);
for (i = 0; i < 256; i++)
@@ -208,7 +209,11 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
hs = (1 << 16) - 1;
else
{
hs = historySize - 1;
hs = historySize;
if (hs > p->expectedDataSize)
hs = (UInt32)p->expectedDataSize;
if (hs != 0)
hs--;
hs |= (hs >> 1);
hs |= (hs >> 2);
hs |= (hs >> 4);
@@ -292,17 +297,33 @@ static void MatchFinder_SetLimits(CMatchFinder *p)
p->posLimit = p->pos + limit;
}
void MatchFinder_Init_2(CMatchFinder *p, int readData)
void MatchFinder_Init_LowHash(CMatchFinder *p)
{
size_t i;
CLzRef *items = p->hash;
size_t numItems = p->fixedHashSize;
for (i = 0; i < numItems; i++)
items[i] = kEmptyHashValue;
}
void MatchFinder_Init_HighHash(CMatchFinder *p)
{
size_t i;
CLzRef *items = p->hash + p->fixedHashSize;
size_t numItems = (size_t)p->hashMask + 1;
for (i = 0; i < numItems; i++)
items[i] = kEmptyHashValue;
}
void MatchFinder_Init_3(CMatchFinder *p, int readData)
{
UInt32 i;
UInt32 *hash = p->hash;
UInt32 num = p->hashSizeSum;
for (i = 0; i < num; i++)
hash[i] = kEmptyHashValue;
p->cyclicBufferPos = 0;
p->buffer = p->bufferBase;
p->pos = p->streamPos = p->cyclicBufferSize;
p->pos =
p->streamPos = p->cyclicBufferSize;
p->result = SZ_OK;
p->streamEndWasReached = 0;
@@ -312,10 +333,14 @@ void MatchFinder_Init_2(CMatchFinder *p, int readData)
MatchFinder_SetLimits(p);
}
void MatchFinder_Init(CMatchFinder *p)
{
MatchFinder_Init_2(p, True);
MatchFinder_Init_HighHash(p);
MatchFinder_Init_LowHash(p);
MatchFinder_Init_3(p, True);
}
static UInt32 MatchFinder_GetSubValue(CMatchFinder *p)
{

8
C/LzFind.h
View File

@@ -1,5 +1,5 @@
/* LzFind.h -- Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2017-06-10 : Igor Pavlov : Public domain */
#ifndef __LZ_FIND_H
#define __LZ_FIND_H
@@ -47,6 +47,8 @@ typedef struct _CMatchFinder
SRes result;
UInt32 crc[256];
size_t numRefs;
UInt64 expectedDataSize;
} CMatchFinder;
#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)
@@ -103,7 +105,9 @@ typedef struct _IMatchFinder
void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable);
void MatchFinder_Init_2(CMatchFinder *p, int readData);
void MatchFinder_Init_LowHash(CMatchFinder *p);
void MatchFinder_Init_HighHash(CMatchFinder *p);
void MatchFinder_Init_3(CMatchFinder *p, int readData);
void MatchFinder_Init(CMatchFinder *p);
UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);

39
C/LzFindMt.c
View File

@@ -1,5 +1,5 @@
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2017-06-10 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -33,6 +33,8 @@ static void MtSync_GetNextBlock(CMtSync *p)
Event_Set(&p->canStart);
Event_Wait(&p->wasStarted);
// if (mt) MatchFinder_Init_LowHash(mt->MatchFinder);
}
else
{
@@ -155,6 +157,9 @@ static void HashThreadFunc(CMatchFinderMt *mt)
UInt32 numProcessedBlocks = 0;
Event_Wait(&p->canStart);
Event_Set(&p->wasStarted);
MatchFinder_Init_HighHash(mt->MatchFinder);
for (;;)
{
if (p->exit)
@@ -205,7 +210,7 @@ static void HashThreadFunc(CMatchFinderMt *mt)
if (num > kMtHashBlockSize - 2)
num = kMtHashBlockSize - 2;
mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc);
heads[0] += num;
heads[0] = 2 + num;
}
mf->pos += num;
mf->buffer += num;
@@ -496,14 +501,18 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB
}
/* Call it after ReleaseStream / SetStream */
void MatchFinderMt_Init(CMatchFinderMt *p)
static void MatchFinderMt_Init(CMatchFinderMt *p)
{
CMatchFinder *mf = p->MatchFinder;
p->btBufPos = p->btBufPosLimit = 0;
p->hashBufPos = p->hashBufPosLimit = 0;
p->btBufPos =
p->btBufPosLimit = 0;
p->hashBufPos =
p->hashBufPosLimit = 0;
/* Init without data reading. We don't want to read data in this thread */
MatchFinder_Init_2(mf, False);
MatchFinder_Init_3(mf, False);
MatchFinder_Init_LowHash(mf);
p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf);
p->btNumAvailBytes = 0;
@@ -684,8 +693,12 @@ static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances)
UInt32 i;
for (i = 0; i < len; i += 2)
{
*distances++ = *btBuf++;
*distances++ = *btBuf++;
UInt32 v0 = btBuf[0];
UInt32 v1 = btBuf[1];
btBuf += 2;
distances[0] = v0;
distances[1] = v1;
distances += 2;
}
}
INCREASE_LZ_POS
@@ -712,8 +725,12 @@ static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances)
distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances);
do
{
*distances2++ = *btBuf++;
*distances2++ = *btBuf++;
UInt32 v0 = btBuf[0];
UInt32 v1 = btBuf[1];
btBuf += 2;
distances2[0] = v0;
distances2[1] = v1;
distances2 += 2;
}
while ((len -= 2) != 0);
len = (UInt32)(distances2 - (distances));
@@ -777,7 +794,7 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable)
{
case 2:
p->GetHeadsFunc = GetHeads2;
p->MixMatchesFunc = (Mf_Mix_Matches)0;
p->MixMatchesFunc = (Mf_Mix_Matches)NULL;
vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip;
vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches;
break;

668
C/Lzma2Enc.c
View File

@@ -1,9 +1,8 @@
/* Lzma2Enc.c -- LZMA2 Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-08-28 : Igor Pavlov : Public domain */
#include "Precomp.h"
/* #include <stdio.h> */
#include <string.h>
/* #define _7ZIP_ST */
@@ -13,7 +12,7 @@
#ifndef _7ZIP_ST
#include "MtCoder.h"
#else
#define NUM_MT_CODER_THREADS_MAX 1
#define MTCODER__THREADS_MAX 1
#endif
#define LZMA2_CONTROL_LZMA (1 << 7)
@@ -35,30 +34,83 @@
#define PRF(x) /* x */
/* ---------- CLimitedSeqInStream ---------- */
typedef struct
{
ISeqInStream vt;
ISeqInStream *realStream;
UInt64 limit;
UInt64 processed;
int finished;
} CLimitedSeqInStream;
static void LimitedSeqInStream_Init(CLimitedSeqInStream *p)
{
p->limit = (UInt64)(Int64)-1;
p->processed = 0;
p->finished = 0;
}
static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size)
{
CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt);
size_t size2 = *size;
SRes res = SZ_OK;
if (p->limit != (UInt64)(Int64)-1)
{
UInt64 rem = p->limit - p->processed;
if (size2 > rem)
size2 = (size_t)rem;
}
if (size2 != 0)
{
res = ISeqInStream_Read(p->realStream, data, &size2);
p->finished = (size2 == 0 ? 1 : 0);
p->processed += size2;
}
*size = size2;
return res;
}
/* ---------- CLzma2EncInt ---------- */
typedef struct
{
CLzmaEncHandle enc;
Byte propsAreSet;
Byte propsByte;
Byte needInitState;
Byte needInitProp;
UInt64 srcPos;
Byte props;
Bool needInitState;
Bool needInitProp;
} CLzma2EncInt;
static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props)
{
Byte propsEncoded[LZMA_PROPS_SIZE];
SizeT propsSize = LZMA_PROPS_SIZE;
RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
p->srcPos = 0;
p->props = propsEncoded[0];
p->needInitState = True;
p->needInitProp = True;
if (!p->propsAreSet)
{
SizeT propsSize = LZMA_PROPS_SIZE;
Byte propsEncoded[LZMA_PROPS_SIZE];
RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
p->propsByte = propsEncoded[0];
p->propsAreSet = True;
}
return SZ_OK;
}
static void Lzma2EncInt_InitBlock(CLzma2EncInt *p)
{
p->srcPos = 0;
p->needInitState = True;
p->needInitProp = True;
}
SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
@@ -70,6 +122,9 @@ void LzmaEnc_Finish(CLzmaEncHandle pp);
void LzmaEnc_SaveState(CLzmaEncHandle pp);
void LzmaEnc_RestoreState(CLzmaEncHandle pp);
/*
UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp);
*/
static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
size_t *packSizeRes, ISeqOutStream *outStream)
@@ -154,7 +209,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
outBuf[destPos++] = (Byte)pm;
if (p->needInitProp)
outBuf[destPos++] = p->props;
outBuf[destPos++] = p->propsByte;
p->needInitProp = False;
p->needInitState = False;
@@ -176,14 +231,16 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
void Lzma2EncProps_Init(CLzma2EncProps *p)
{
LzmaEncProps_Init(&p->lzmaProps);
p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO;
p->numBlockThreads_Reduced = -1;
p->numBlockThreads_Max = -1;
p->numTotalThreads = -1;
p->numBlockThreads = -1;
p->blockSize = 0;
}
void Lzma2EncProps_Normalize(CLzma2EncProps *p)
{
int t1, t1n, t2, t3;
UInt64 fileSize;
int t1, t1n, t2, t2r, t3;
{
CLzmaEncProps lzmaProps = p->lzmaProps;
LzmaEncProps_Normalize(&lzmaProps);
@@ -191,11 +248,11 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
}
t1 = p->lzmaProps.numThreads;
t2 = p->numBlockThreads;
t2 = p->numBlockThreads_Max;
t3 = p->numTotalThreads;
if (t2 > NUM_MT_CODER_THREADS_MAX)
t2 = NUM_MT_CODER_THREADS_MAX;
if (t2 > MTCODER__THREADS_MAX)
t2 = MTCODER__THREADS_MAX;
if (t3 <= 0)
{
@@ -211,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
t1 = 1;
t2 = t3;
}
if (t2 > NUM_MT_CODER_THREADS_MAX)
t2 = NUM_MT_CODER_THREADS_MAX;
if (t2 > MTCODER__THREADS_MAX)
t2 = MTCODER__THREADS_MAX;
}
else if (t1 <= 0)
{
@@ -225,39 +282,64 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
p->lzmaProps.numThreads = t1;
t2r = t2;
fileSize = p->lzmaProps.reduceSize;
if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
&& p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
&& (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1))
p->lzmaProps.reduceSize = p->blockSize;
LzmaEncProps_Normalize(&p->lzmaProps);
p->lzmaProps.reduceSize = fileSize;
t1 = p->lzmaProps.numThreads;
if (p->blockSize == 0)
if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID)
{
UInt32 dictSize = p->lzmaProps.dictSize;
UInt64 blockSize = (UInt64)dictSize << 2;
const UInt32 kMinSize = (UInt32)1 << 20;
const UInt32 kMaxSize = (UInt32)1 << 28;
if (blockSize < kMinSize) blockSize = kMinSize;
if (blockSize > kMaxSize) blockSize = kMaxSize;
if (blockSize < dictSize) blockSize = dictSize;
p->blockSize = (size_t)blockSize;
t2r = t2 = 1;
t3 = t1;
}
if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)
else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1)
{
UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
if (temp > p->lzmaProps.reduceSize)
/* if there is no block multi-threading, we use SOLID block */
p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID;
}
else
{
if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO)
{
UInt64 numBlocks = temp / p->blockSize;
const UInt32 kMinSize = (UInt32)1 << 20;
const UInt32 kMaxSize = (UInt32)1 << 28;
const UInt32 dictSize = p->lzmaProps.dictSize;
UInt64 blockSize = (UInt64)dictSize << 2;
if (blockSize < kMinSize) blockSize = kMinSize;
if (blockSize > kMaxSize) blockSize = kMaxSize;
if (blockSize < dictSize) blockSize = dictSize;
blockSize += (kMinSize - 1);
blockSize &= ~(UInt64)(kMinSize - 1);
p->blockSize = blockSize;
}
if (t2 > 1 && fileSize != (UInt64)(Int64)-1)
{
UInt64 numBlocks = fileSize / p->blockSize;
if (numBlocks * p->blockSize != fileSize)
numBlocks++;
if (numBlocks < (unsigned)t2)
{
t2 = (unsigned)numBlocks;
if (t2 == 0)
t2 = 1;
t3 = t1 * t2;
t2r = (unsigned)numBlocks;
if (t2r == 0)
t2r = 1;
t3 = t1 * t2r;
}
}
}
p->numBlockThreads = t2;
p->numBlockThreads_Max = t2;
p->numBlockThreads_Reduced = t2r;
p->numTotalThreads = t3;
}
@@ -274,134 +356,30 @@ typedef struct
{
Byte propEncoded;
CLzma2EncProps props;
UInt64 expectedDataSize;
Byte *outBuf;
Byte *tempBufLzma;
ISzAllocPtr alloc;
ISzAllocPtr allocBig;
CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
CLzma2EncInt coders[MTCODER__THREADS_MAX];
#ifndef _7ZIP_ST
ISeqOutStream *outStream;
Byte *outBuf;
size_t outBufSize;
size_t outBufsDataSizes[MTCODER__BLOCKS_MAX];
Bool mtCoder_WasConstructed;
CMtCoder mtCoder;
Byte *outBufs[MTCODER__BLOCKS_MAX];
#endif
} CLzma2Enc;
/* ---------- Lzma2EncThread ---------- */
static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
UInt64 packTotal = 0;
SRes res = SZ_OK;
if (!mainEncoder->outBuf)
{
mainEncoder->outBuf = (Byte *)ISzAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
if (!mainEncoder->outBuf)
return SZ_ERROR_MEM;
}
RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
mainEncoder->alloc, mainEncoder->allocBig));
for (;;)
{
size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
if (res != SZ_OK)
break;
packTotal += packSize;
res = Progress(progress, p->srcPos, packTotal);
if (res != SZ_OK)
break;
if (packSize == 0)
break;
}
LzmaEnc_Finish(p->enc);
if (res == SZ_OK)
{
Byte b = 0;
if (ISeqOutStream_Write(outStream, &b, 1) != 1)
return SZ_ERROR_WRITE;
}
return res;
}
#ifndef _7ZIP_ST
typedef struct
{
IMtCoderCallback funcTable;
CLzma2Enc *lzma2Enc;
} CMtCallbackImp;
static SRes MtCallbackImp_Code(const IMtCoderCallback *pp, unsigned index, Byte *dest, size_t *destSize,
const Byte *src, size_t srcSize, int finished)
{
CMtCallbackImp *imp = CONTAINER_FROM_VTBL(pp, CMtCallbackImp, funcTable);
CLzma2Enc *mainEncoder = imp->lzma2Enc;
CLzma2EncInt *p = &mainEncoder->coders[index];
SRes res = SZ_OK;
{
size_t destLim = *destSize;
*destSize = 0;
if (srcSize != 0)
{
RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
mainEncoder->alloc, mainEncoder->allocBig));
while (p->srcPos < srcSize)
{
size_t packSize = destLim - *destSize;
res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
if (res != SZ_OK)
break;
*destSize += packSize;
if (packSize == 0)
{
res = SZ_ERROR_FAIL;
break;
}
if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
{
res = SZ_ERROR_PROGRESS;
break;
}
}
LzmaEnc_Finish(p->enc);
if (res != SZ_OK)
return res;
}
if (finished)
{
if (*destSize == destLim)
return SZ_ERROR_OUTPUT_EOF;
dest[(*destSize)++] = 0;
}
}
return res;
}
#endif
/* ---------- Lzma2Enc ---------- */
CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)
{
@@ -410,44 +388,78 @@ CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)
return NULL;
Lzma2EncProps_Init(&p->props);
Lzma2EncProps_Normalize(&p->props);
p->outBuf = 0;
p->expectedDataSize = (UInt64)(Int64)-1;
p->tempBufLzma = NULL;
p->alloc = alloc;
p->allocBig = allocBig;
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->coders[i].enc = 0;
for (i = 0; i < MTCODER__THREADS_MAX; i++)
p->coders[i].enc = NULL;
}
#ifndef _7ZIP_ST
MtCoder_Construct(&p->mtCoder);
p->mtCoder_WasConstructed = False;
{
unsigned i;
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
p->outBufs[i] = NULL;
p->outBufSize = 0;
}
#endif
return p;
}
#ifndef _7ZIP_ST
static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p)
{
unsigned i;
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
if (p->outBufs[i])
{
ISzAlloc_Free(p->alloc, p->outBufs[i]);
p->outBufs[i] = NULL;
}
p->outBufSize = 0;
}
#endif
void Lzma2Enc_Destroy(CLzma2EncHandle pp)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CLzma2EncInt *t = &p->coders[i];
if (t->enc)
{
LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
t->enc = 0;
t->enc = NULL;
}
}
#ifndef _7ZIP_ST
MtCoder_Destruct(&p->mtCoder);
if (p->mtCoder_WasConstructed)
{
MtCoder_Destruct(&p->mtCoder);
p->mtCoder_WasConstructed = False;
}
Lzma2Enc_FreeOutBufs(p);
#endif
ISzAlloc_Free(p->alloc, p->outBuf);
ISzAlloc_Free(p->alloc, p->tempBufLzma);
p->tempBufLzma = NULL;
ISzAlloc_Free(p->alloc, pp);
}
SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
@@ -460,6 +472,14 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
return SZ_OK;
}
void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
p->expectedDataSize = expectedDataSiize;
}
Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
@@ -471,50 +491,310 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
return (Byte)i;
}
SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
int i;
for (i = 0; i < p->props.numBlockThreads; i++)
static SRes Lzma2Enc_EncodeMt1(
CLzma2Enc *me,
CLzma2EncInt *p,
ISeqOutStream *outStream,
Byte *outBuf, size_t *outBufSize,
ISeqInStream *inStream,
const Byte *inData, size_t inDataSize,
int finished,
ICompressProgress *progress)
{
UInt64 unpackTotal = 0;
UInt64 packTotal = 0;
size_t outLim = 0;
CLimitedSeqInStream limitedInStream;
if (outBuf)
{
CLzma2EncInt *t = &p->coders[(unsigned)i];
if (!t->enc)
outLim = *outBufSize;
*outBufSize = 0;
}
if (!p->enc)
{
p->propsAreSet = False;
p->enc = LzmaEnc_Create(me->alloc);
if (!p->enc)
return SZ_ERROR_MEM;
}
limitedInStream.realStream = inStream;
if (inStream)
{
limitedInStream.vt.Read = LimitedSeqInStream_Read;
}
if (!outBuf)
{
// outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma
if (!me->tempBufLzma)
{
t->enc = LzmaEnc_Create(p->alloc);
if (!t->enc)
me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
if (!me->tempBufLzma)
return SZ_ERROR_MEM;
}
}
#ifndef _7ZIP_ST
if (p->props.numBlockThreads > 1)
{
CMtCallbackImp mtCallback;
RINOK(Lzma2EncInt_InitStream(p, &me->props));
mtCallback.funcTable.Code = MtCallbackImp_Code;
mtCallback.lzma2Enc = p;
for (;;)
{
SRes res = SZ_OK;
size_t inSizeCur = 0;
Lzma2EncInt_InitBlock(p);
LimitedSeqInStream_Init(&limitedInStream);
limitedInStream.limit = me->props.blockSize;
if (inStream)
{
UInt64 expected = (UInt64)(Int64)-1;
// inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize
if (me->expectedDataSize != (UInt64)(Int64)-1
&& me->expectedDataSize >= unpackTotal)
expected = me->expectedDataSize - unpackTotal;
if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
&& expected > me->props.blockSize)
expected = (size_t)me->props.blockSize;
LzmaEnc_SetDataSize(p->enc, expected);
RINOK(LzmaEnc_PrepareForLzma2(p->enc,
&limitedInStream.vt,
LZMA2_KEEP_WINDOW_SIZE,
me->alloc,
me->allocBig));
}
else
{
inSizeCur = inDataSize - (size_t)unpackTotal;
if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
&& inSizeCur > me->props.blockSize)
inSizeCur = (size_t)me->props.blockSize;
// LzmaEnc_SetDataSize(p->enc, inSizeCur);
RINOK(LzmaEnc_MemPrepare(p->enc,
inData + (size_t)unpackTotal, inSizeCur,
LZMA2_KEEP_WINDOW_SIZE,
me->alloc,
me->allocBig));
}
for (;;)
{
size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
if (outBuf)
packSize = outLim - (size_t)packTotal;
res = Lzma2EncInt_EncodeSubblock(p,
outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize,
outBuf ? NULL : outStream);
if (res != SZ_OK)
break;
packTotal += packSize;
if (outBuf)
*outBufSize = (size_t)packTotal;
res = Progress(progress, unpackTotal + p->srcPos, packTotal);
if (res != SZ_OK)
break;
/*
if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0)
break;
*/
if (packSize == 0)
break;
}
LzmaEnc_Finish(p->enc);
unpackTotal += p->srcPos;
RINOK(res);
if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur))
return SZ_ERROR_FAIL;
if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize))
{
if (finished)
{
if (outBuf)
{
size_t destPos = *outBufSize;
if (destPos >= outLim)
return SZ_ERROR_OUTPUT_EOF;
outBuf[destPos] = 0;
*outBufSize = destPos + 1;
}
else
{
Byte b = 0;
if (ISeqOutStream_Write(outStream, &b, 1) != 1)
return SZ_ERROR_WRITE;
}
}
return SZ_OK;
}
}
}
#ifndef _7ZIP_ST
static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex,
const Byte *src, size_t srcSize, int finished)
{
CLzma2Enc *me = (CLzma2Enc *)pp;
size_t destSize = me->outBufSize;
SRes res;
CMtProgressThunk progressThunk;
Byte *dest = me->outBufs[outBufIndex];
me->outBufsDataSizes[outBufIndex] = 0;
if (!dest)
{
dest = ISzAlloc_Alloc(me->alloc, me->outBufSize);
if (!dest)
return SZ_ERROR_MEM;
me->outBufs[outBufIndex] = dest;
}
MtProgressThunk_CreateVTable(&progressThunk);
progressThunk.mtProgress = &me->mtCoder.mtProgress;
progressThunk.index = coderIndex;
res = Lzma2Enc_EncodeMt1(me,
&me->coders[coderIndex],
NULL, dest, &destSize,
NULL, src, srcSize,
finished,
&progressThunk.vt);
me->outBufsDataSizes[outBufIndex] = destSize;
return res;
}
static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex)
{
CLzma2Enc *me = (CLzma2Enc *)pp;
size_t size = me->outBufsDataSizes[outBufIndex];
const Byte *data = me->outBufs[outBufIndex];
if (me->outStream)
return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE;
if (size > me->outBufSize)
return SZ_ERROR_OUTPUT_EOF;
memcpy(me->outBuf, data, size);
me->outBufSize -= size;
me->outBuf += size;
return SZ_OK;
}
#endif
SRes Lzma2Enc_Encode2(CLzma2EncHandle pp,
ISeqOutStream *outStream,
Byte *outBuf, size_t *outBufSize,
ISeqInStream *inStream,
const Byte *inData, size_t inDataSize,
ICompressProgress *progress)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
if (inStream && inData)
return E_INVALIDARG;
if (outStream && outBuf)
return E_INVALIDARG;
{
unsigned i;
for (i = 0; i < MTCODER__THREADS_MAX; i++)
p->coders[i].propsAreSet = False;
}
#ifndef _7ZIP_ST
if (p->props.numBlockThreads_Reduced > 1)
{
IMtCoderCallback2 vt;
if (!p->mtCoder_WasConstructed)
{
p->mtCoder_WasConstructed = True;
MtCoder_Construct(&p->mtCoder);
}
vt.Code = Lzma2Enc_MtCallback_Code;
vt.Write = Lzma2Enc_MtCallback_Write;
p->outStream = outStream;
p->outBuf = NULL;
p->outBufSize = 0;
if (!outStream)
{
p->outBuf = outBuf;
p->outBufSize = *outBufSize;
*outBufSize = 0;
}
p->mtCoder.allocBig = p->allocBig;
p->mtCoder.progress = progress;
p->mtCoder.inStream = inStream;
p->mtCoder.outStream = outStream;
p->mtCoder.alloc = p->alloc;
p->mtCoder.mtCallback = &mtCallback.funcTable;
p->mtCoder.inData = inData;
p->mtCoder.inDataSize = inDataSize;
p->mtCoder.mtCallback = &vt;
p->mtCoder.mtCallbackObject = p;
p->mtCoder.blockSize = (size_t)p->props.blockSize;
if (p->mtCoder.blockSize != p->props.blockSize)
return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */
p->mtCoder.blockSize = p->props.blockSize;
p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
if (p->mtCoder.destBlockSize < p->props.blockSize)
{
p->mtCoder.destBlockSize = (size_t)0 - 1;
if (p->mtCoder.destBlockSize < p->props.blockSize)
return SZ_ERROR_FAIL;
size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16;
if (destBlockSize < p->mtCoder.blockSize)
return SZ_ERROR_PARAM;
if (p->outBufSize != destBlockSize)
Lzma2Enc_FreeOutBufs(p);
p->outBufSize = destBlockSize;
}
p->mtCoder.numThreads = p->props.numBlockThreads;
p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max;
p->mtCoder.expectedDataSize = p->expectedDataSize;
return MtCoder_Code(&p->mtCoder);
{
SRes res = MtCoder_Code(&p->mtCoder);
if (!outStream)
*outBufSize = p->outBuf - outBuf;
return res;
}
}
#endif
return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
return Lzma2Enc_EncodeMt1(p,
&p->coders[0],
outStream, outBuf, outBufSize,
inStream, inData, inDataSize,
True, /* finished */
progress);
}

43
C/Lzma2Enc.h
View File

@@ -1,5 +1,5 @@
/* Lzma2Enc.h -- LZMA2 Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#ifndef __LZMA2_ENC_H
#define __LZMA2_ENC_H
@@ -8,11 +8,15 @@
EXTERN_C_BEGIN
#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0
#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1)
typedef struct
{
CLzmaEncProps lzmaProps;
size_t blockSize;
int numBlockThreads;
UInt64 blockSize;
int numBlockThreads_Reduced;
int numBlockThreads_Max;
int numTotalThreads;
} CLzma2EncProps;
@@ -22,13 +26,14 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p);
/* ---------- CLzmaEnc2Handle Interface ---------- */
/* Lzma2Enc_* functions can return the following exit codes:
Returns:
SRes:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater in props
SZ_ERROR_WRITE - Write callback error
SZ_ERROR_WRITE - ISeqOutStream write callback error
SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
SZ_ERROR_PROGRESS - some break from progress callback
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
*/
typedef void * CLzma2EncHandle;
@@ -36,26 +41,14 @@ typedef void * CLzma2EncHandle;
CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig);
void Lzma2Enc_Destroy(CLzma2EncHandle p);
SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props);
void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize);
Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p);
SRes Lzma2Enc_Encode(CLzma2EncHandle p,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
/* ---------- One Call Interface ---------- */
/* Lzma2Encode
Return code:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater
SZ_ERROR_OUTPUT_EOF - output buffer overflow
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
/*
SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark,
ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
*/
SRes Lzma2Enc_Encode2(CLzma2EncHandle p,
ISeqOutStream *outStream,
Byte *outBuf, size_t *outBufSize,
ISeqInStream *inStream,
const Byte *inData, size_t inDataSize,
ICompressProgress *progress);
EXTERN_C_END

47
C/LzmaEnc.c
View File

@@ -1,5 +1,5 @@
/* LzmaEnc.c -- LZMA Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-06-22 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -23,8 +23,8 @@
static unsigned g_STAT_OFFSET = 0;
#endif
#define kMaxHistorySize ((UInt32)3 << 29)
/* #define kMaxHistorySize ((UInt32)7 << 29) */
#define kLzmaMaxHistorySize ((UInt32)3 << 29)
/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */
#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1)
@@ -62,14 +62,15 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p)
if (level < 0) level = 5;
p->level = level;
if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26)));
if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26)));
if (p->dictSize > p->reduceSize)
{
unsigned i;
UInt32 reduceSize = (UInt32)p->reduceSize;
for (i = 11; i <= 30; i++)
{
if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; }
if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; }
if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; }
if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; }
}
}
@@ -445,7 +446,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
|| props.lp > LZMA_LP_MAX
|| props.pb > LZMA_PB_MAX
|| props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress)
|| props.dictSize > kMaxHistorySize)
|| props.dictSize > kLzmaMaxHistorySize)
return SZ_ERROR_PARAM;
p->dictSize = props.dictSize;
@@ -492,6 +493,15 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
return SZ_OK;
}
void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize)
{
CLzmaEnc *p = (CLzmaEnc *)pp;
p->matchFinderBase.expectedDataSize = expectedDataSiize;
}
static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5};
static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10};
static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11};
@@ -1996,6 +2006,8 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc,
{
RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig));
p->matchFinderObj = &p->matchFinderMt;
p->matchFinderBase.bigHash = (Byte)(
(p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0);
MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder);
}
else
@@ -2135,6 +2147,7 @@ SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
LzmaEnc_SetInputBuf(p, src, srcLen);
p->needInit = 1;
LzmaEnc_SetDataSize(pp, srcLen);
return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig);
}
@@ -2152,15 +2165,15 @@ void LzmaEnc_Finish(CLzmaEncHandle pp)
typedef struct
{
ISeqOutStream funcTable;
ISeqOutStream vt;
Byte *data;
SizeT rem;
Bool overflow;
} CSeqOutStreamBuf;
} CLzmaEnc_SeqOutStreamBuf;
static size_t MyWrite(const ISeqOutStream *pp, const void *data, size_t size)
static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size)
{
CSeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CSeqOutStreamBuf, funcTable);
CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt);
if (p->rem < size)
{
size = p->rem;
@@ -2193,9 +2206,9 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
CLzmaEnc *p = (CLzmaEnc *)pp;
UInt64 nowPos64;
SRes res;
CSeqOutStreamBuf outStream;
CLzmaEnc_SeqOutStreamBuf outStream;
outStream.funcTable.Write = MyWrite;
outStream.vt.Write = SeqOutStreamBuf_Write;
outStream.data = dest;
outStream.rem = *destLen;
outStream.overflow = False;
@@ -2209,7 +2222,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
LzmaEnc_InitPrices(p);
nowPos64 = p->nowPos64;
RangeEnc_Init(&p->rc);
p->rc.outStream = &outStream.funcTable;
p->rc.outStream = &outStream.vt;
res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize);
@@ -2308,15 +2321,15 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte
SRes res;
CLzmaEnc *p = (CLzmaEnc *)pp;
CSeqOutStreamBuf outStream;
CLzmaEnc_SeqOutStreamBuf outStream;
outStream.funcTable.Write = MyWrite;
outStream.vt.Write = SeqOutStreamBuf_Write;
outStream.data = dest;
outStream.rem = *destLen;
outStream.overflow = False;
p->writeEndMark = writeEndMark;
p->rc.outStream = &outStream.funcTable;
p->rc.outStream = &outStream.vt;
res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig);

37
C/LzmaEnc.h
View File

@@ -1,5 +1,5 @@
/* LzmaEnc.h -- LZMA Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#ifndef __LZMA_ENC_H
#define __LZMA_ENC_H
@@ -12,12 +12,10 @@ EXTERN_C_BEGIN
typedef struct _CLzmaEncProps
{
int level; /* 0 <= level <= 9 */
int level; /* 0 <= level <= 9 */
UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version
(1 << 12) <= dictSize <= (1 << 30) for 64-bit version
default = (1 << 24) */
UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF.
Encoder uses this value to reduce dictionary size */
(1 << 12) <= dictSize <= (3 << 29) for 64-bit version
default = (1 << 24) */
int lc; /* 0 <= lc <= 8, default = 3 */
int lp; /* 0 <= lp <= 4, default = 0 */
int pb; /* 0 <= pb <= 4, default = 2 */
@@ -25,9 +23,12 @@ typedef struct _CLzmaEncProps
int fb; /* 5 <= fb <= 273, default = 32 */
int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */
int numHashBytes; /* 2, 3 or 4, default = 4 */
UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */
UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */
unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */
int numThreads; /* 1 or 2, default = 2 */
UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1.
Encoder uses this value to reduce dictionary size */
} CLzmaEncProps;
void LzmaEncProps_Init(CLzmaEncProps *p);
@@ -37,38 +38,34 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2);
/* ---------- CLzmaEncHandle Interface ---------- */
/* LzmaEnc_* functions can return the following exit codes:
Returns:
/* LzmaEnc* functions can return the following exit codes:
SRes:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater in props
SZ_ERROR_WRITE - Write callback error.
SZ_ERROR_WRITE - ISeqOutStream write callback error
SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
SZ_ERROR_PROGRESS - some break from progress callback
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
*/
typedef void * CLzmaEncHandle;
CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc);
void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props);
void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize);
SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size);
unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p);
SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream,
ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
/* ---------- One Call Interface ---------- */
/* LzmaEncode
Return code:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater
SZ_ERROR_OUTPUT_EOF - output buffer overflow
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
/* ---------- One Call Interface ---------- */
SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark,

745
C/MtCoder.c
View File

@@ -1,95 +1,72 @@
/* MtCoder.c -- Multi-thread Coder
2017-04-03 : Igor Pavlov : Public domain */
2017-07-17 : Igor Pavlov : Public domain */
#include "Precomp.h"
#include "MtCoder.h"
void LoopThread_Construct(CLoopThread *p)
{
Thread_Construct(&p->thread);
Event_Construct(&p->startEvent);
Event_Construct(&p->finishedEvent);
}
void LoopThread_Close(CLoopThread *p)
{
Thread_Close(&p->thread);
Event_Close(&p->startEvent);
Event_Close(&p->finishedEvent);
}
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp)
{
CLoopThread *p = (CLoopThread *)pp;
for (;;)
{
if (Event_Wait(&p->startEvent) != 0)
return SZ_ERROR_THREAD;
if (p->stop)
return 0;
p->res = p->func(p->param);
if (Event_Set(&p->finishedEvent) != 0)
return SZ_ERROR_THREAD;
}
}
WRes LoopThread_Create(CLoopThread *p)
{
p->stop = 0;
RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent));
RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent));
return Thread_Create(&p->thread, LoopThreadFunc, p);
}
WRes LoopThread_StopAndWait(CLoopThread *p)
{
p->stop = 1;
if (Event_Set(&p->startEvent) != 0)
return SZ_ERROR_THREAD;
return Thread_Wait(&p->thread);
}
WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); }
WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); }
static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
{
return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
}
static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress)
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->inSizes[i] = p->outSizes[i] = 0;
p->totalInSize = p->totalOutSize = 0;
p->progress = progress;
p->res = SZ_OK;
p->totalInSize = 0;
p->totalOutSize = 0;
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CMtProgressSizes *pair = &p->sizes[i];
pair->inSize = 0;
pair->outSize = 0;
}
}
static void MtProgress_Reinit(CMtProgress *p, unsigned index)
{
p->inSizes[index] = 0;
p->outSizes[index] = 0;
CMtProgressSizes *pair = &p->sizes[index];
pair->inSize = 0;
pair->outSize = 0;
}
#define UPDATE_PROGRESS(size, prev, total) \
if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; }
SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize)
{
SRes res;
CMtProgressSizes *pair;
CriticalSection_Enter(&p->cs);
pair = &p->sizes[index];
UPDATE_PROGRESS(inSize, pair->inSize, p->totalInSize)
UPDATE_PROGRESS(outSize, pair->outSize, p->totalOutSize)
if (p->res == SZ_OK && p->progress)
{
if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK)
p->res = SZ_ERROR_PROGRESS;
}
res = p->res;
CriticalSection_Leave(&p->cs);
return res;
}
static SRes MtProgress_GetError(CMtProgress *p)
{
SRes res;
CriticalSection_Enter(&p->cs);
UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize)
UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize)
if (p->res == SZ_OK)
p->res = Progress(p->progress, p->totalInSize, p->totalOutSize);
res = p->res;
CriticalSection_Leave(&p->cs);
return res;
}
static void MtProgress_SetError(CMtProgress *p, SRes res)
{
CriticalSection_Enter(&p->cs);
@@ -98,71 +75,72 @@ static void MtProgress_SetError(CMtProgress *p, SRes res)
CriticalSection_Leave(&p->cs);
}
static void MtCoder_SetError(CMtCoder* p, SRes res)
static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize)
{
CriticalSection_Enter(&p->cs);
if (p->res == SZ_OK)
p->res = res;
CriticalSection_Leave(&p->cs);
CMtProgressThunk *p = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt);
return MtProgress_Set(p->mtProgress, p->index, inSize, outSize);
}
/* ---------- MtThread ---------- */
void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder)
void MtProgressThunk_CreateVTable(CMtProgressThunk *p)
{
p->mtCoder = mtCoder;
p->outBuf = 0;
p->inBuf = 0;
Event_Construct(&p->canRead);
Event_Construct(&p->canWrite);
LoopThread_Construct(&p->thread);
p->vt.Progress = MtProgressThunk_Progress;
}
#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; }
static void CMtThread_CloseEvents(CMtThread *p)
static WRes ArEvent_OptCreate_And_Reset(CEvent *p)
{
Event_Close(&p->canRead);
Event_Close(&p->canWrite);
if (Event_IsCreated(p))
return Event_Reset(p);
return AutoResetEvent_CreateNotSignaled(p);
}
static void CMtThread_Destruct(CMtThread *p)
{
CMtThread_CloseEvents(p);
if (Thread_WasCreated(&p->thread.thread))
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp);
static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t)
{
WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent);
if (wres == 0)
{
LoopThread_StopAndWait(&p->thread);
LoopThread_Close(&p->thread);
t->stop = False;
if (!Thread_WasCreated(&t->thread))
wres = Thread_Create(&t->thread, ThreadFunc, t);
if (wres == 0)
wres = Event_Set(&t->startEvent);
}
if (wres == 0)
return SZ_OK;
return MY_SRes_HRESULT_FROM_WRes(wres);
}
static void MtCoderThread_Destruct(CMtCoderThread *t)
{
if (Thread_WasCreated(&t->thread))
{
t->stop = 1;
Event_Set(&t->startEvent);
Thread_Wait(&t->thread);
Thread_Close(&t->thread);
}
if (p->mtCoder->alloc)
ISzAlloc_Free(p->mtCoder->alloc, p->outBuf);
p->outBuf = 0;
Event_Close(&t->startEvent);
if (p->mtCoder->alloc)
ISzAlloc_Free(p->mtCoder->alloc, p->inBuf);
p->inBuf = 0;
if (t->inBuf)
{
ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf);
t->inBuf = NULL;
}
}
#define MY_BUF_ALLOC(buf, size, newSize) \
if (buf == 0 || size != newSize) \
{ ISzAlloc_Free(p->mtCoder->alloc, buf); \
size = newSize; buf = (Byte *)ISzAlloc_Alloc(p->mtCoder->alloc, size); \
if (buf == 0) return SZ_ERROR_MEM; }
static SRes CMtThread_Prepare(CMtThread *p)
{
MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize)
MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize)
p->stopReading = False;
p->stopWriting = False;
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead));
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite));
return SZ_OK;
}
static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
{
@@ -170,158 +148,509 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
*processedSize = 0;
while (size != 0)
{
size_t curSize = size;
SRes res = ISeqInStream_Read(stream, data, &curSize);
*processedSize += curSize;
data += curSize;
size -= curSize;
size_t cur = size;
SRes res = ISeqInStream_Read(stream, data, &cur);
*processedSize += cur;
data += cur;
size -= cur;
RINOK(res);
if (curSize == 0)
if (cur == 0)
return SZ_OK;
}
return SZ_OK;
}
#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1]
static SRes MtThread_Process(CMtThread *p, Bool *stop)
/*
ThreadFunc2() returns:
SZ_OK - in all normal cases (even for stream error or memory allocation error)
SZ_ERROR_THREAD - in case of failure in system synch function
*/
static SRes ThreadFunc2(CMtCoderThread *t)
{
CMtThread *next;
*stop = True;
if (Event_Wait(&p->canRead) != 0)
return SZ_ERROR_THREAD;
next = GET_NEXT_THREAD(p);
if (p->stopReading)
{
next->stopReading = True;
return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD;
}
CMtCoder *mtc = t->mtCoder;
{
size_t size = p->mtCoder->blockSize;
size_t destSize = p->outBufSize;
RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size));
next->stopReading = *stop = (size != p->mtCoder->blockSize);
if (Event_Set(&next->canRead) != 0)
return SZ_ERROR_THREAD;
RINOK(IMtCoderCallback_Code(p->mtCoder->mtCallback, p->index,
p->outBuf, &destSize, p->inBuf, size, *stop));
MtProgress_Reinit(&p->mtCoder->mtProgress, p->index);
if (Event_Wait(&p->canWrite) != 0)
return SZ_ERROR_THREAD;
if (p->stopWriting)
return SZ_ERROR_FAIL;
if (ISeqOutStream_Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize)
return SZ_ERROR_WRITE;
return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD;
}
}
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
{
CMtThread *p = (CMtThread *)pp;
for (;;)
{
Bool stop;
CMtThread *next = GET_NEXT_THREAD(p);
SRes res = MtThread_Process(p, &stop);
if (res != SZ_OK)
unsigned bi;
SRes res;
SRes res2;
Bool finished;
unsigned bufIndex;
size_t size;
const Byte *inData;
UInt64 readProcessed = 0;
RINOK_THREAD(Event_Wait(&mtc->readEvent))
/* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */
if (mtc->stopReading)
{
MtCoder_SetError(p->mtCoder, res);
MtProgress_SetError(&p->mtCoder->mtProgress, res);
next->stopReading = True;
next->stopWriting = True;
Event_Set(&next->canRead);
Event_Set(&next->canWrite);
return res;
return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD;
}
if (stop)
res = MtProgress_GetError(&mtc->mtProgress);
size = 0;
inData = NULL;
finished = True;
if (res == SZ_OK)
{
size = mtc->blockSize;
if (mtc->inStream)
{
if (!t->inBuf)
{
t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize);
if (!t->inBuf)
res = SZ_ERROR_MEM;
}
if (res == SZ_OK)
{
res = FullRead(mtc->inStream, t->inBuf, &size);
readProcessed = mtc->readProcessed + size;
mtc->readProcessed = readProcessed;
}
if (res != SZ_OK)
{
mtc->readRes = res;
/* after reading error - we can stop encoding of previous blocks */
MtProgress_SetError(&mtc->mtProgress, res);
}
else
finished = (size != mtc->blockSize);
}
else
{
size_t rem;
readProcessed = mtc->readProcessed;
rem = mtc->inDataSize - (size_t)readProcessed;
if (size > rem)
size = rem;
inData = mtc->inData + (size_t)readProcessed;
readProcessed += size;
mtc->readProcessed = readProcessed;
finished = (mtc->inDataSize == (size_t)readProcessed);
}
}
/* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */
res2 = SZ_OK;
if (Semaphore_Wait(&mtc->blocksSemaphore) != 0)
{
res2 = SZ_ERROR_THREAD;
if (res == SZ_OK)
{
res = res2;
// MtProgress_SetError(&mtc->mtProgress, res);
}
}
bi = mtc->blockIndex;
if (++mtc->blockIndex >= mtc->numBlocksMax)
mtc->blockIndex = 0;
bufIndex = (unsigned)(int)-1;
if (res == SZ_OK)
res = MtProgress_GetError(&mtc->mtProgress);
if (res != SZ_OK)
finished = True;
if (!finished)
{
if (mtc->numStartedThreads < mtc->numStartedThreadsLimit
&& mtc->expectedDataSize != readProcessed)
{
res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]);
if (res == SZ_OK)
mtc->numStartedThreads++;
else
{
MtProgress_SetError(&mtc->mtProgress, res);
finished = True;
}
}
}
if (finished)
mtc->stopReading = True;
RINOK_THREAD(Event_Set(&mtc->readEvent))
if (res2 != SZ_OK)
return res2;
if (res == SZ_OK)
{
CriticalSection_Enter(&mtc->cs);
bufIndex = mtc->freeBlockHead;
mtc->freeBlockHead = mtc->freeBlockList[bufIndex];
CriticalSection_Leave(&mtc->cs);
res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex,
mtc->inStream ? t->inBuf : inData, size, finished);
MtProgress_Reinit(&mtc->mtProgress, t->index);
if (res != SZ_OK)
MtProgress_SetError(&mtc->mtProgress, res);
}
{
CMtCoderBlock *block = &mtc->blocks[bi];
block->res = res;
block->bufIndex = bufIndex;
block->finished = finished;
}
#ifdef MTCODER__USE_WRITE_THREAD
RINOK_THREAD(Event_Set(&mtc->writeEvents[bi]))
#else
{
unsigned wi;
{
CriticalSection_Enter(&mtc->cs);
wi = mtc->writeIndex;
if (wi == bi)
mtc->writeIndex = (unsigned)(int)-1;
else
mtc->ReadyBlocks[bi] = True;
CriticalSection_Leave(&mtc->cs);
}
if (wi != bi)
{
if (res != SZ_OK || finished)
return 0;
continue;
}
if (mtc->writeRes != SZ_OK)
res = mtc->writeRes;
for (;;)
{
if (res == SZ_OK && bufIndex != (unsigned)(int)-1)
{
res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex);
if (res != SZ_OK)
{
mtc->writeRes = res;
MtProgress_SetError(&mtc->mtProgress, res);
}
}
if (++wi >= mtc->numBlocksMax)
wi = 0;
{
Bool isReady;
CriticalSection_Enter(&mtc->cs);
if (bufIndex != (unsigned)(int)-1)
{
mtc->freeBlockList[bufIndex] = mtc->freeBlockHead;
mtc->freeBlockHead = bufIndex;
}
isReady = mtc->ReadyBlocks[wi];
if (isReady)
mtc->ReadyBlocks[wi] = False;
else
mtc->writeIndex = wi;
CriticalSection_Leave(&mtc->cs);
RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore))
if (!isReady)
break;
}
{
CMtCoderBlock *block = &mtc->blocks[wi];
if (res == SZ_OK && block->res != SZ_OK)
res = block->res;
bufIndex = block->bufIndex;
finished = block->finished;
}
}
}
#endif
if (finished || res != SZ_OK)
return 0;
}
}
void MtCoder_Construct(CMtCoder* p)
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
{
CMtCoderThread *t = (CMtCoderThread *)pp;
for (;;)
{
if (Event_Wait(&t->startEvent) != 0)
return SZ_ERROR_THREAD;
if (t->stop)
return 0;
{
SRes res = ThreadFunc2(t);
CMtCoder *mtc = t->mtCoder;
if (res != SZ_OK)
{
MtProgress_SetError(&mtc->mtProgress, res);
}
#ifndef MTCODER__USE_WRITE_THREAD
{
unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads);
if (numFinished == mtc->numStartedThreads)
if (Event_Set(&mtc->finishedEvent) != 0)
return SZ_ERROR_THREAD;
}
#endif
}
}
}
void MtCoder_Construct(CMtCoder *p)
{
unsigned i;
p->alloc = 0;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->blockSize = 0;
p->numThreadsMax = 0;
p->expectedDataSize = (UInt64)(Int64)-1;
p->inStream = NULL;
p->inData = NULL;
p->inDataSize = 0;
p->progress = NULL;
p->allocBig = NULL;
p->mtCallback = NULL;
p->mtCallbackObject = NULL;
p->allocatedBufsSize = 0;
Event_Construct(&p->readEvent);
Semaphore_Construct(&p->blocksSemaphore);
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CMtThread *t = &p->threads[i];
CMtCoderThread *t = &p->threads[i];
t->mtCoder = p;
t->index = i;
CMtThread_Construct(t, p);
t->inBuf = NULL;
t->stop = False;
Event_Construct(&t->startEvent);
Thread_Construct(&t->thread);
}
#ifdef MTCODER__USE_WRITE_THREAD
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
Event_Construct(&p->writeEvents[i]);
#else
Event_Construct(&p->finishedEvent);
#endif
CriticalSection_Init(&p->cs);
CriticalSection_Init(&p->mtProgress.cs);
}
void MtCoder_Destruct(CMtCoder* p)
static void MtCoder_Free(CMtCoder *p)
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
CMtThread_Destruct(&p->threads[i]);
/*
p->stopReading = True;
if (Event_IsCreated(&p->readEvent))
Event_Set(&p->readEvent);
*/
for (i = 0; i < MTCODER__THREADS_MAX; i++)
MtCoderThread_Destruct(&p->threads[i]);
Event_Close(&p->readEvent);
Semaphore_Close(&p->blocksSemaphore);
#ifdef MTCODER__USE_WRITE_THREAD
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
Event_Close(&p->writeEvents[i]);
#else
Event_Close(&p->finishedEvent);
#endif
}
void MtCoder_Destruct(CMtCoder *p)
{
MtCoder_Free(p);
CriticalSection_Delete(&p->cs);
CriticalSection_Delete(&p->mtProgress.cs);
}
SRes MtCoder_Code(CMtCoder *p)
{
unsigned i, numThreads = p->numThreads;
unsigned numThreads = p->numThreadsMax;
unsigned numBlocksMax;
unsigned i;
SRes res = SZ_OK;
p->res = SZ_OK;
if (numThreads > MTCODER__THREADS_MAX)
numThreads = MTCODER__THREADS_MAX;
numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads);
if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++;
if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++;
if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++;
if (numBlocksMax > MTCODER__BLOCKS_MAX)
numBlocksMax = MTCODER__BLOCKS_MAX;
if (p->blockSize != p->allocatedBufsSize)
{
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CMtCoderThread *t = &p->threads[i];
if (t->inBuf)
{
ISzAlloc_Free(p->allocBig, t->inBuf);
t->inBuf = NULL;
}
}
p->allocatedBufsSize = p->blockSize;
}
p->readRes = SZ_OK;
MtProgress_Init(&p->mtProgress, p->progress);
for (i = 0; i < numThreads; i++)
#ifdef MTCODER__USE_WRITE_THREAD
for (i = 0; i < numBlocksMax; i++)
{
RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i]));
}
#else
RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent));
#endif
{
RINOK(CMtThread_Prepare(&p->threads[i]));
RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent));
if (Semaphore_IsCreated(&p->blocksSemaphore))
{
RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore));
}
RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax));
}
for (i = 0; i < numThreads; i++)
for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++)
p->freeBlockList[i] = i + 1;
p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1;
p->freeBlockHead = 0;
p->readProcessed = 0;
p->blockIndex = 0;
p->numBlocksMax = numBlocksMax;
p->stopReading = False;
#ifndef MTCODER__USE_WRITE_THREAD
p->writeIndex = 0;
p->writeRes = SZ_OK;
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
p->ReadyBlocks[i] = False;
p->numFinishedThreads = 0;
#endif
p->numStartedThreadsLimit = numThreads;
p->numStartedThreads = 0;
// for (i = 0; i < numThreads; i++)
{
CMtThread *t = &p->threads[i];
CLoopThread *lt = &t->thread;
CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++];
RINOK(MtCoderThread_CreateAndStart(nextThread));
}
if (!Thread_WasCreated(&lt->thread))
RINOK_THREAD(Event_Set(&p->readEvent))
#ifdef MTCODER__USE_WRITE_THREAD
{
unsigned bi = 0;
for (;; bi++)
{
lt->func = ThreadFunc;
lt->param = t;
if (bi >= numBlocksMax)
bi = 0;
RINOK_THREAD(Event_Wait(&p->writeEvents[bi]))
if (LoopThread_Create(lt) != SZ_OK)
{
res = SZ_ERROR_THREAD;
break;
const CMtCoderBlock *block = &p->blocks[bi];
unsigned bufIndex = block->bufIndex;
Bool finished = block->finished;
if (res == SZ_OK && block->res != SZ_OK)
res = block->res;
if (bufIndex != (unsigned)(int)-1)
{
if (res == SZ_OK)
{
res = p->mtCallback->Write(p->mtCallbackObject, bufIndex);
if (res != SZ_OK)
MtProgress_SetError(&p->mtProgress, res);
}
CriticalSection_Enter(&p->cs);
{
p->freeBlockList[bufIndex] = p->freeBlockHead;
p->freeBlockHead = bufIndex;
}
CriticalSection_Leave(&p->cs);
}
RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore))
if (finished)
break;
}
}
}
#else
{
WRes wres = Event_Wait(&p->finishedEvent);
res = MY_SRes_HRESULT_FROM_WRes(wres);
}
#endif
if (res == SZ_OK)
{
unsigned j;
for (i = 0; i < numThreads; i++)
{
CMtThread *t = &p->threads[i];
if (LoopThread_StartSubThread(&t->thread) != SZ_OK)
{
res = SZ_ERROR_THREAD;
p->threads[0].stopReading = True;
break;
}
}
res = p->readRes;
Event_Set(&p->threads[0].canWrite);
Event_Set(&p->threads[0].canRead);
if (res == SZ_OK)
res = p->mtProgress.res;
for (j = 0; j < i; j++)
LoopThread_WaitSubThread(&p->threads[j].thread);
}
#ifndef MTCODER__USE_WRITE_THREAD
if (res == SZ_OK)
res = p->writeRes;
#endif
for (i = 0; i < numThreads; i++)
CMtThread_CloseEvents(&p->threads[i]);
return (res == SZ_OK) ? p->res : res;
if (res != SZ_OK)
MtCoder_Free(p);
return res;
}

155
C/MtCoder.h
View File

@@ -1,5 +1,5 @@
/* MtCoder.h -- Multi-thread Coder
2017-04-03 : Igor Pavlov : Public domain */
2017-06-18 : Igor Pavlov : Public domain */
#ifndef __MT_CODER_H
#define __MT_CODER_H
@@ -8,95 +8,144 @@
EXTERN_C_BEGIN
typedef struct
{
CThread thread;
CAutoResetEvent startEvent;
CAutoResetEvent finishedEvent;
int stop;
THREAD_FUNC_TYPE func;
LPVOID param;
THREAD_FUNC_RET_TYPE res;
} CLoopThread;
void LoopThread_Construct(CLoopThread *p);
void LoopThread_Close(CLoopThread *p);
WRes LoopThread_Create(CLoopThread *p);
WRes LoopThread_StopAndWait(CLoopThread *p);
WRes LoopThread_StartSubThread(CLoopThread *p);
WRes LoopThread_WaitSubThread(CLoopThread *p);
/*
if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream
if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream
*/
/* #define MTCODER__USE_WRITE_THREAD */
#ifndef _7ZIP_ST
#define NUM_MT_CODER_THREADS_MAX 32
#define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1)
#define MTCODER__THREADS_MAX 64
#define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3)
#else
#define NUM_MT_CODER_THREADS_MAX 1
#define MTCODER__THREADS_MAX 1
#define MTCODER__BLOCKS_MAX 1
#endif
typedef struct
{
UInt64 inSize;
UInt64 outSize;
} CMtProgressSizes;
typedef struct
{
UInt64 totalInSize;
UInt64 totalOutSize;
ICompressProgress *progress;
SRes res;
UInt64 totalInSize;
UInt64 totalOutSize;
CCriticalSection cs;
UInt64 inSizes[NUM_MT_CODER_THREADS_MAX];
UInt64 outSizes[NUM_MT_CODER_THREADS_MAX];
CMtProgressSizes sizes[MTCODER__THREADS_MAX];
} CMtProgress;
SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize);
typedef struct
{
ICompressProgress vt;
CMtProgress *mtProgress;
unsigned index;
} CMtProgressThunk;
void MtProgressThunk_CreateVTable(CMtProgressThunk *p);
struct _CMtCoder;
typedef struct
{
struct _CMtCoder *mtCoder;
Byte *outBuf;
size_t outBufSize;
Byte *inBuf;
size_t inBufSize;
unsigned index;
CLoopThread thread;
int stop;
Byte *inBuf;
Bool stopReading;
Bool stopWriting;
CAutoResetEvent canRead;
CAutoResetEvent canWrite;
} CMtThread;
CAutoResetEvent startEvent;
CThread thread;
} CMtCoderThread;
typedef struct IMtCoderCallback IMtCoderCallback;
struct IMtCoderCallback
typedef struct
{
SRes (*Code)(const IMtCoderCallback *p, unsigned index, Byte *dest, size_t *destSize,
SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex,
const Byte *src, size_t srcSize, int finished);
};
#define IMtCoderCallback_Code(p, index, dest, destSize, src, srcSize, finished) (p)->Code(p, index, dest, destSize, src, srcSize, finished)
SRes (*Write)(void *p, unsigned outBufIndex);
} IMtCoderCallback2;
typedef struct
{
SRes res;
unsigned bufIndex;
Bool finished;
} CMtCoderBlock;
typedef struct _CMtCoder
{
size_t blockSize;
size_t destBlockSize;
unsigned numThreads;
/* input variables */
ISeqInStream *inStream;
ISeqOutStream *outStream;
ICompressProgress *progress;
ISzAllocPtr alloc;
size_t blockSize; /* size of input block */
unsigned numThreadsMax;
UInt64 expectedDataSize;
ISeqInStream *inStream;
const Byte *inData;
size_t inDataSize;
ICompressProgress *progress;
ISzAllocPtr allocBig;
IMtCoderCallback2 *mtCallback;
void *mtCallbackObject;
/* internal variables */
size_t allocatedBufsSize;
CAutoResetEvent readEvent;
CSemaphore blocksSemaphore;
Bool stopReading;
SRes readRes;
#ifdef MTCODER__USE_WRITE_THREAD
CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX];
#else
CAutoResetEvent finishedEvent;
SRes writeRes;
unsigned writeIndex;
Byte ReadyBlocks[MTCODER__BLOCKS_MAX];
LONG numFinishedThreads;
#endif
unsigned numStartedThreadsLimit;
unsigned numStartedThreads;
unsigned numBlocksMax;
unsigned blockIndex;
UInt64 readProcessed;
IMtCoderCallback *mtCallback;
CCriticalSection cs;
SRes res;
unsigned freeBlockHead;
unsigned freeBlockList[MTCODER__BLOCKS_MAX];
CMtProgress mtProgress;
CMtThread threads[NUM_MT_CODER_THREADS_MAX];
CMtCoderBlock blocks[MTCODER__BLOCKS_MAX];
CMtCoderThread threads[MTCODER__THREADS_MAX];
} CMtCoder;
void MtCoder_Construct(CMtCoder* p);
void MtCoder_Destruct(CMtCoder* p);
void MtCoder_Construct(CMtCoder *p);
void MtCoder_Destruct(CMtCoder *p);
SRes MtCoder_Code(CMtCoder *p);
EXTERN_C_END
#endif

14
C/Threads.c
View File

@@ -1,5 +1,5 @@
/* Threads.c -- multithreading library
2014-09-21 : Igor Pavlov : Public domain */
2017-06-26 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -12,18 +12,20 @@
static WRes GetError()
{
DWORD res = GetLastError();
return (res) ? (WRes)(res) : 1;
return res ? (WRes)res : 1;
}
WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); }
WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); }
static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
WRes HandlePtr_Close(HANDLE *p)
{
if (*p != NULL)
{
if (!CloseHandle(*p))
return GetError();
*p = NULL;
*p = NULL;
}
return 0;
}
@@ -49,7 +51,7 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
return HandleToWRes(*p);
}
WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
{
*p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL);
return HandleToWRes(*p);

5
C/Threads.h
View File

@@ -1,5 +1,5 @@
/* Threads.h -- multithreading library
2013-11-12 : Igor Pavlov : Public domain */
2017-06-18 : Igor Pavlov : Public domain */
#ifndef __7Z_THREADS_H
#define __7Z_THREADS_H
@@ -49,7 +49,8 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled);
WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p);
typedef HANDLE CSemaphore;
#define Semaphore_Construct(p) (*p) = NULL
#define Semaphore_Construct(p) *(p) = NULL
#define Semaphore_IsCreated(p) (*(p) != NULL)
#define Semaphore_Close(p) HandlePtr_Close(p)
#define Semaphore_Wait(p) Handle_WaitObject(*(p))
WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount);

11
C/Util/7z/7zMain.c
View File

@@ -1,5 +1,5 @@
/* 7zMain.c - Test application for 7z Decoder
2017-04-05 : Igor Pavlov : Public domain */
2017-08-26 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -616,7 +616,14 @@ int MY_CDECL main(int numargs, char *args[])
#ifdef USE_WINDOWS_FILE
if (SzBitWithVals_Check(&db.Attribs, i))
SetFileAttributesW(destPath, db.Attribs.Vals[i]);
{
UInt32 attrib = db.Attribs.Vals[i];
/* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker.
We remove posix bits, if we detect posix mode field */
if ((attrib & 0xF0000000) != 0)
attrib &= 0x7FFF;
SetFileAttributesW(destPath, attrib);
}
#endif
}
PrintLF();

4
C/Util/7zipInstall/7zipInstall.c
View File

@@ -1,5 +1,5 @@
/* 7zipInstall.c - 7-Zip Installer
2017-04-04 : Igor Pavlov : Public domain */
2017-08-28 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -154,7 +154,7 @@ static WRes MyCreateDir(LPCWSTR name)
}
#define IS_SEPAR(c) (c == WCHAR_PATH_SEPARATOR)
#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z')
#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
#define IS_DRIVE_PATH(s) (IS_LETTER_CHAR(s[0]) && s[1] == ':' && IS_SEPAR(s[2]))
static int ReverseFind_PathSepar(const wchar_t *s)

8
C/Util/Lzma/LzmaUtil.dsp
View File

@@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c
# End Source File
# Begin Source File
SOURCE=..\..\7zTypes.h
# End Source File
# Begin Source File
SOURCE=..\..\7zVersion.h
# End Source File
# Begin Source File
@@ -160,9 +164,5 @@ SOURCE=..\..\Threads.c
SOURCE=..\..\Threads.h
# End Source File
# Begin Source File
SOURCE=..\..\Types.h
# End Source File
# End Target
# End Project

12
C/Xz.c
View File

@@ -1,5 +1,5 @@
/* Xz.c - Xz
2017-04-03 : Igor Pavlov : Public domain */
2017-05-12 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -9,7 +9,7 @@
#include "XzCrc64.h"
const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 };
const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' };
/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */
unsigned Xz_WriteVarInt(Byte *buf, UInt64 v)
{
@@ -26,16 +26,16 @@ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v)
void Xz_Construct(CXzStream *p)
{
p->numBlocks = p->numBlocksAllocated = 0;
p->blocks = 0;
p->numBlocks = 0;
p->blocks = NULL;
p->flags = 0;
}
void Xz_Free(CXzStream *p, ISzAllocPtr alloc)
{
ISzAlloc_Free(alloc, p->blocks);
p->numBlocks = p->numBlocksAllocated = 0;
p->blocks = 0;
p->numBlocks = 0;
p->blocks = NULL;
}
unsigned XzFlags_GetCheckSize(CXzStreamFlags f)

35
C/Xz.h
View File

@@ -1,5 +1,5 @@
/* Xz.h - Xz interface
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#ifndef __XZ_H
#define __XZ_H
@@ -50,6 +50,7 @@ typedef struct
#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1)
#define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0)
#define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0)
#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0)
SRes XzBlock_Parse(CXzBlock *p, const Byte *header);
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes);
@@ -60,7 +61,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
#define XZ_FOOTER_SIG_SIZE 2
extern const Byte XZ_SIG[XZ_SIG_SIZE];
/*
extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE];
*/
#define XZ_FOOTER_SIG_0 'Y'
#define XZ_FOOTER_SIG_1 'Z'
#define XZ_STREAM_FLAGS_SIZE 2
#define XZ_STREAM_CRC_SIZE 4
@@ -106,7 +113,6 @@ typedef struct
{
CXzStreamFlags flags;
size_t numBlocks;
size_t numBlocksAllocated;
CXzBlockSizes *blocks;
UInt64 startOffset;
} CXzStream;
@@ -218,6 +224,9 @@ typedef struct
CXzBlock block;
CXzCheck check;
CSha256 sha;
unsigned decodeOnlyOneBlock;
Byte shaDigest[SHA256_DIGEST_SIZE];
Byte buf[XZ_BLOCK_HEADER_SIZE_MAX];
} CXzUnpacker;
@@ -258,7 +267,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode,
ECoderStatus *status);
Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p);
Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p);
/*
Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of
@@ -268,7 +277,25 @@ XzUnpacker_Code() returns:
res == SZ_ERROR_NO_ARCHIVE
*/
UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p);
UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p);
/*
for random block decoding:
XzUnpacker_Init();
set CXzUnpacker::streamFlags
XzUnpacker_PrepareToRandomBlockDecoding()
loop
{
XzUnpacker_Code()
XzUnpacker_IsBlockFinished()
}
*/
void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p);
Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p);
#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags))
EXTERN_C_END

16
C/XzCrc64.c
View File

@@ -1,5 +1,5 @@
/* XzCrc64.c -- CRC64 calculation
2017-04-03 : Igor Pavlov : Public domain */
2017-05-23 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -9,9 +9,9 @@
#define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42)
#ifdef MY_CPU_LE
#define CRC_NUM_TABLES 4
#define CRC64_NUM_TABLES 4
#else
#define CRC_NUM_TABLES 5
#define CRC64_NUM_TABLES 5
#define CRC_UINT64_SWAP(v) \
((v >> 56) \
| ((v >> 40) & ((UInt64)0xFF << 8)) \
@@ -29,10 +29,10 @@
UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table);
#endif
typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table);
typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table);
static CRC_FUNC g_Crc64Update;
UInt64 g_Crc64Table[256 * CRC_NUM_TABLES];
static CRC64_FUNC g_Crc64Update;
UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES];
UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size)
{
@@ -55,7 +55,7 @@ void MY_FAST_CALL Crc64GenerateTable()
r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1)));
g_Crc64Table[i] = r;
}
for (i = 256; i < 256 * CRC_NUM_TABLES; i++)
for (i = 256; i < 256 * CRC64_NUM_TABLES; i++)
{
UInt64 r = g_Crc64Table[(size_t)i - 256];
g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8);
@@ -74,7 +74,7 @@ void MY_FAST_CALL Crc64GenerateTable()
else
#endif
{
for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--)
for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--)
{
UInt64 x = g_Crc64Table[(size_t)i - 256];
g_Crc64Table[i] = CRC_UINT64_SWAP(x);

14
C/XzCrc64Opt.c
View File

@@ -1,5 +1,5 @@
/* XzCrc64Opt.c -- CRC64 calculation
2017-04-03 : Igor Pavlov : Public domain */
2017-06-30 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -7,13 +7,13 @@
#ifndef MY_CPU_BE
#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table)
{
const Byte *p = (const Byte *)data;
for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++)
v = CRC_UPDATE_BYTE_2(v, *p);
v = CRC64_UPDATE_BYTE_2(v, *p);
for (; size >= 4; size -= 4, p += 4)
{
UInt32 d = (UInt32)v ^ *(const UInt32 *)p;
@@ -24,7 +24,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con
^ (table + 0x000)[((d >> 24))];
}
for (; size > 0; size--, p++)
v = CRC_UPDATE_BYTE_2(v, *p);
v = CRC64_UPDATE_BYTE_2(v, *p);
return v;
}
@@ -43,7 +43,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con
| ((v << 40) & ((UInt64)0xFF << 48)) \
| ((v << 56)))
#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8))
#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8))
UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table)
{
@@ -51,7 +51,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size
table += 0x100;
v = CRC_UINT64_SWAP(v);
for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++)
v = CRC_UPDATE_BYTE_2_BE(v, *p);
v = CRC64_UPDATE_BYTE_2_BE(v, *p);
for (; size >= 4; size -= 4, p += 4)
{
UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p;
@@ -62,7 +62,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size
^ (table + 0x300)[((d >> 24))];
}
for (; size > 0; size--, p++)
v = CRC_UPDATE_BYTE_2_BE(v, *p);
v = CRC64_UPDATE_BYTE_2_BE(v, *p);
return CRC_UINT64_SWAP(v);
}

151
C/XzDec.c
View File

@@ -1,5 +1,5 @@
/* XzDec.c -- Xz Decode
2017-04-03 : Igor Pavlov : Public domain */
2018-01-21 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -393,7 +393,7 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
{
SizeT destLenOrig = *destLen;
SizeT srcLenOrig = *srcLen;
Bool allFinished = True;
*destLen = 0;
*srcLen = 0;
*status = CODER_STATUS_NOT_FINISHED;
@@ -411,6 +411,7 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
for (;;)
{
Bool processed = False;
Bool allFinished = True;
unsigned i;
/*
if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY)
@@ -487,14 +488,17 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
if (destLenCur != 0 || srcLenCur != 0)
processed = True;
}
if (!processed)
break;
{
if (allFinished)
*status = CODER_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
}
if (allFinished)
*status = CODER_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf)
{
*p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE);
@@ -506,11 +510,11 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf)
static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf)
{
return
indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) &&
(GetUi32(buf) == CrcCalc(buf + 4, 6) &&
flags == GetBe16(buf + 8) &&
memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0);
return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2)
&& GetUi32(buf) == CrcCalc(buf + 4, 6)
&& flags == GetBe16(buf + 8)
&& buf[10] == XZ_FOOTER_SIG_0
&& buf[11] == XZ_FOOTER_SIG_1;
}
#define READ_VARINT_AND_CHECK(buf, pos, size, res) \
@@ -524,14 +528,15 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
unsigned numFilters, i;
unsigned headerSize = (unsigned)header[0] << 2;
/* (headerSize != 0) : another code checks */
if (CrcCalc(header, headerSize) != GetUi32(header + headerSize))
return SZ_ERROR_ARCHIVE;
pos = 1;
if (pos == headerSize)
return SZ_ERROR_ARCHIVE;
p->flags = header[pos++];
p->packSize = (UInt64)(Int64)-1;
if (XzBlock_HasPackSize(p))
{
READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize);
@@ -539,6 +544,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
return SZ_ERROR_ARCHIVE;
}
p->unpackSize = (UInt64)(Int64)-1;
if (XzBlock_HasUnpackSize(p))
READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize);
@@ -565,6 +571,9 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
#endif
}
if (XzBlock_HasUnsupportedFlags(p))
return SZ_ERROR_UNSUPPORTED;
while (pos < headerSize)
if (header[pos++] != 0)
return SZ_ERROR_ARCHIVE;
@@ -615,6 +624,7 @@ void XzUnpacker_Init(CXzUnpacker *p)
p->numFinishedStreams = 0;
p->numTotalBlocks = 0;
p->padSize = 0;
p->decodeOnlyOneBlock = 0;
}
void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc)
@@ -628,6 +638,18 @@ void XzUnpacker_Free(CXzUnpacker *p)
MixCoder_Free(&p->decoder);
}
void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p)
{
p->indexSize = 0;
p->numBlocks = 0;
Sha256_Init(&p->sha);
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
p->decodeOnlyOneBlock = 1;
}
SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status)
{
@@ -638,20 +660,44 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
*status = CODER_STATUS_NOT_SPECIFIED;
for (;;)
{
SizeT srcRem = srcLenOrig - *srcLen;
SizeT srcRem;
if (p->state == XZ_STATE_BLOCK)
{
SizeT destLen2 = destLenOrig - *destLen;
SizeT srcLen2 = srcLenOrig - *srcLen;
SRes res;
ECoderFinishMode finishMode2 = finishMode;
if (p->block.packSize != (UInt64)(Int64)-1)
{
UInt64 rem = p->block.packSize - p->packSize;
if (srcLen2 > rem)
srcLen2 = (SizeT)rem;
if (rem == 0 && p->block.unpackSize == p->unpackSize)
return SZ_ERROR_DATA;
}
if (p->block.unpackSize != (UInt64)(Int64)-1)
{
UInt64 rem = p->block.unpackSize - p->unpackSize;
if (destLen2 >= rem)
{
finishMode2 = CODER_FINISH_END;
destLen2 = (SizeT)rem;
}
}
/*
if (srcLen2 == 0 && destLen2 == 0)
{
*status = CODER_STATUS_NOT_FINISHED;
return SZ_OK;
}
*/
res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode, status);
res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode2, status);
XzCheck_Update(&p->check, dest, destLen2);
(*srcLen) += srcLen2;
@@ -661,29 +707,43 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
(*destLen) += destLen2;
dest += destLen2;
p->unpackSize += destLen2;
RINOK(res);
if (*status == CODER_STATUS_FINISHED_WITH_MARK)
if (*status != CODER_STATUS_FINISHED_WITH_MARK)
{
if (p->block.packSize == p->packSize
&& *status == CODER_STATUS_NEEDS_MORE_INPUT)
{
*status = CODER_STATUS_NOT_SPECIFIED;
return SZ_ERROR_DATA;
}
// if (srcLen2 == 0 && destLen2 == 0)
return SZ_OK;
}
{
Byte temp[32];
unsigned num = Xz_WriteVarInt(temp, p->packSize + p->blockHeaderSize + XzFlags_GetCheckSize(p->streamFlags));
unsigned num = Xz_WriteVarInt(temp, XzUnpacker_GetPackSizeForIndex(p));
num += Xz_WriteVarInt(temp + num, p->unpackSize);
Sha256_Update(&p->sha, temp, num);
p->indexSize += num;
p->numBlocks++;
p->state = XZ_STATE_BLOCK_FOOTER;
p->pos = 0;
p->alignPos = 0;
if (p->block.unpackSize != (UInt64)(Int64)-1)
if (p->block.unpackSize != p->unpackSize)
return SZ_ERROR_DATA;
}
else if (srcLen2 == 0 && destLen2 == 0)
return SZ_OK;
continue;
// continue;
}
if (srcRem == 0)
srcRem = srcLenOrig - *srcLen;
// XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes
if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER)
{
*status = CODER_STATUS_NEEDS_MORE_INPUT;
return SZ_OK;
@@ -704,10 +764,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
{
RINOK(Xz_ParseHeader(&p->streamFlags, p->buf));
p->numStartedStreams++;
p->state = XZ_STATE_BLOCK_HEADER;
Sha256_Init(&p->sha);
p->indexSize = 0;
p->numBlocks = 0;
Sha256_Init(&p->sha);
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
}
break;
@@ -721,6 +781,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
(*srcLen)++;
if (p->buf[0] == 0)
{
if (p->decodeOnlyOneBlock)
return SZ_ERROR_DATA;
p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks);
p->indexPos = p->indexPreSize;
p->indexSize += p->indexPreSize;
@@ -728,10 +790,13 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
Sha256_Init(&p->sha);
p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize);
p->state = XZ_STATE_STREAM_INDEX;
break;
}
p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4;
break;
}
else if (p->pos != p->blockHeaderSize)
if (p->pos != p->blockHeaderSize)
{
UInt32 cur = p->blockHeaderSize - p->pos;
if (cur > srcRem)
@@ -756,8 +821,13 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
case XZ_STATE_BLOCK_FOOTER:
{
if (((p->packSize + p->alignPos) & 3) != 0)
if ((((unsigned)p->packSize + p->alignPos) & 3) != 0)
{
if (srcRem == 0)
{
*status = CODER_STATUS_NEEDS_MORE_INPUT;
return SZ_OK;
}
(*srcLen)++;
p->alignPos++;
if (*src++ != 0)
@@ -769,20 +839,31 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
UInt32 cur = checkSize - p->pos;
if (cur != 0)
{
if (srcRem == 0)
{
*status = CODER_STATUS_NEEDS_MORE_INPUT;
return SZ_OK;
}
if (cur > srcRem)
cur = (UInt32)srcRem;
memcpy(p->buf + p->pos, src, cur);
p->pos += cur;
(*srcLen) += cur;
src += cur;
if (checkSize != p->pos)
break;
}
else
{
Byte digest[XZ_CHECK_SIZE_MAX];
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0)
return SZ_ERROR_CRC;
if (p->decodeOnlyOneBlock)
{
*status = CODER_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
}
}
break;
@@ -898,12 +979,18 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
*/
}
Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p)
Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p)
{
return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0);
}
Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p)
{
return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0);
}
UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p)
UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p)
{
UInt64 num = 0;
if (p->state == XZ_STATE_STREAM_PADDING)

1264
C/XzEnc.c
View File

File diff suppressed because it is too large Load Diff

27
C/XzEnc.h
View File

@@ -1,5 +1,5 @@
/* XzEnc.h -- Xz Encode
2011-02-07 : Igor Pavlov : Public domain */
2017-06-27 : Igor Pavlov : Public domain */
#ifndef __XZ_ENC_H
#define __XZ_ENC_H
@@ -10,6 +10,11 @@
EXTERN_C_BEGIN
#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
typedef struct
{
UInt32 id;
@@ -20,15 +25,31 @@ typedef struct
void XzFilterProps_Init(CXzFilterProps *p);
typedef struct
{
const CLzma2EncProps *lzma2Props;
const CXzFilterProps *filterProps;
CLzma2EncProps lzma2Props;
CXzFilterProps filterProps;
unsigned checkId;
UInt64 blockSize;
int numBlockThreads_Reduced;
int numBlockThreads_Max;
int numTotalThreads;
int forceWriteSizesInHeader;
UInt64 reduceSize;
} CXzProps;
void XzProps_Init(CXzProps *p);
typedef void * CXzEncHandle;
CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig);
void XzEnc_Destroy(CXzEncHandle p);
SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props);
void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize);
SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
const CXzProps *props, ICompressProgress *progress);

32
C/XzIn.c
View File

@@ -1,5 +1,5 @@
/* XzIn.c - Xz input
2017-04-03 : Igor Pavlov : Public domain */
2017-05-11 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -9,6 +9,12 @@
#include "CpuArch.h"
#include "Xz.h"
/*
#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0)
*/
#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1)
SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream)
{
Byte sig[XZ_STREAM_HEADER_SIZE];
@@ -28,7 +34,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
unsigned headerSize;
*headerSizeRes = 0;
RINOK(SeqInStream_ReadByte(inStream, &header[0]));
headerSize = ((unsigned)header[0] << 2) + 4;
headerSize = (unsigned)header[0];
if (headerSize == 0)
{
*headerSizeRes = 1;
@@ -37,12 +43,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
}
*isIndex = False;
headerSize = (headerSize << 2) + 4;
*headerSizeRes = headerSize;
RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1));
return XzBlock_Parse(p, header);
}
#define ADD_SIZE_CHECH(size, val) \
#define ADD_SIZE_CHECK(size, val) \
{ UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; }
UInt64 Xz_GetUnpackSize(const CXzStream *p)
@@ -50,7 +57,7 @@ UInt64 Xz_GetUnpackSize(const CXzStream *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->numBlocks; i++)
ADD_SIZE_CHECH(size, p->blocks[i].unpackSize);
ADD_SIZE_CHECK(size, p->blocks[i].unpackSize);
return size;
}
@@ -59,7 +66,7 @@ UInt64 Xz_GetPackSize(const CXzStream *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->numBlocks; i++)
ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3);
ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3);
return size;
}
@@ -96,9 +103,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt
{
size_t i;
p->numBlocks = numBlocks;
p->numBlocksAllocated = numBlocks;
p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks);
if (p->blocks == 0)
if (!p->blocks)
return SZ_ERROR_MEM;
for (i = 0; i < numBlocks; i++)
{
@@ -126,7 +132,7 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize,
if (size != indexSize)
return SZ_ERROR_UNSUPPORTED;
buf = ISzAlloc_Alloc(alloc, size);
if (buf == 0)
if (!buf)
return SZ_ERROR_MEM;
res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED);
if (res == SZ_OK)
@@ -154,7 +160,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff
pos -= XZ_STREAM_FOOTER_SIZE;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE));
if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0)
if (!XZ_FOOTER_SIG_CHECK(buf + 10))
{
UInt32 total = 0;
pos += XZ_STREAM_FOOTER_SIZE;
@@ -187,7 +193,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff
return SZ_ERROR_NO_ARCHIVE;
pos -= XZ_STREAM_FOOTER_SIZE;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE));
if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0)
if (!XZ_FOOTER_SIG_CHECK(buf + 10))
return SZ_ERROR_NO_ARCHIVE;
}
@@ -262,7 +268,7 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->num; i++)
ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i]));
ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i]));
return size;
}
@@ -272,7 +278,7 @@ UInt64 Xzs_GetPackSize(const CXzs *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->num; i++)
ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i]));
ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i]));
return size;
}
*/
@@ -294,7 +300,7 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr
{
size_t newNum = p->num + p->num / 4 + 1;
Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream));
if (data == 0)
if (!data)
return SZ_ERROR_MEM;
p->numAllocated = newNum;
if (p->num != 0)

13
CPP/7zip/Archive/7z/7zEncode.cpp
View File

@@ -370,6 +370,17 @@ HRESULT CEncoder::Encode(
resetInitVector->ResetInitVector();
}
{
CMyComPtr<ICompressSetCoderPropertiesOpt> optProps;
coder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps);
if (optProps)
{
PROPID propID = NCoderPropID::kExpectedDataSize;
NWindows::NCOM::CPropVariant prop = (UInt64)unpackSize;
RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1));
}
}
CMyComPtr<ICompressWriteCoderProperties> writeCoderProperties;
coder->QueryInterface(IID_ICompressWriteCoderProperties, (void **)&writeCoderProperties);
@@ -380,7 +391,7 @@ HRESULT CEncoder::Encode(
CDynBufSeqOutStream *outStreamSpec = new CDynBufSeqOutStream;
CMyComPtr<ISequentialOutStream> dynOutStream(outStreamSpec);
outStreamSpec->Init();
writeCoderProperties->WriteCoderProperties(dynOutStream);
RINOK(writeCoderProperties->WriteCoderProperties(dynOutStream));
outStreamSpec->CopyToBuffer(props);
}
else

15
CPP/7zip/Archive/7z/7zFolderInStream.cpp
View File

@@ -80,14 +80,17 @@ STDMETHODIMP CFolderInStream::Read(void *data, UInt32 size, UInt32 *processedSiz
{
if (_stream)
{
UInt32 processed2;
RINOK(_stream->Read(data, size, &processed2));
if (processed2 != 0)
UInt32 cur = size;
const UInt32 kMax = (UInt32)1 << 20;
if (cur > kMax)
cur = kMax;
RINOK(_stream->Read(data, cur, &cur));
if (cur != 0)
{
_crc = CrcUpdate(_crc, data, processed2);
_pos += processed2;
_crc = CrcUpdate(_crc, data, cur);
_pos += cur;
if (processedSize)
*processedSize = processed2;
*processedSize = cur;
return S_OK;
}

2
CPP/7zip/Archive/7z/7zOut.cpp
View File

@@ -515,7 +515,7 @@ HRESULT COutArchive::EncodeStream(
outFolders.FolderUnpackCRCs.Vals.Add(CrcCalc(data, data.Size()));
// outFolders.NumUnpackStreamsVector.Add(1);
UInt64 dataSize64 = data.Size();
UInt64 unpackSize;
UInt64 unpackSize = data.Size();
RINOK(encoder.Encode(
EXTERNAL_CODECS_LOC_VARS
stream,

5
CPP/7zip/Archive/7z/7zUpdate.cpp
View File

@@ -2171,6 +2171,7 @@ HRESULT Update(
#endif
}
curUnpackSize = sizeToEncode;
HRESULT encodeRes = encoder.Encode(
EXTERNAL_CODECS_LOC_VARS
@@ -2358,7 +2359,9 @@ HRESULT Update(
inStreamSpec->Init(updateCallback, &indices[i], numSubFiles);
unsigned startPackIndex = newDatabase.PackSizes.Size();
UInt64 curFolderUnpackSize;
UInt64 curFolderUnpackSize = totalSize;
// curFolderUnpackSize = (UInt64)(Int64)-1;
RINOK(encoder.Encode(
EXTERNAL_CODECS_LOC_VARS
solidInStream,

34
CPP/7zip/Archive/DmgHandler.cpp
View File

@@ -19,6 +19,7 @@
#include "../Compress/BZip2Decoder.h"
#include "../Compress/CopyCoder.h"
#include "../Compress/LzfseDecoder.h"
#include "../Compress/ZlibDecoder.h"
#include "Common/OutStreamWithCRC.h"
@@ -121,6 +122,7 @@ enum
METHOD_ADC = 0x80000004,
METHOD_ZLIB = 0x80000005,
METHOD_BZIP2 = 0x80000006,
METHOD_LZFSE = 0x80000007,
METHOD_COMMENT = 0x7FFFFFFE, // is used to comment "+beg" and "+end" in extra field.
METHOD_END = 0xFFFFFFFF
};
@@ -276,6 +278,7 @@ void CMethods::GetString(AString &res) const
case METHOD_ADC: s = "ADC"; break;
case METHOD_ZLIB: s = "ZLIB"; break;
case METHOD_BZIP2: s = "BZip2"; break;
case METHOD_LZFSE: s = "LZFSE"; break;
default: ConvertUInt32ToString(type, buf); s = buf;
}
res.Add_OptSpaced(s);
@@ -307,6 +310,10 @@ static const CAppleName k_Names[] =
{ true, "hfs", "Apple_HFS" },
{ true, "hfsx", "Apple_HFSX" },
{ true, "ufs", "Apple_UFS" },
// efi_sys partition is FAT32, but it's not main file. So we use (IsFs = false)
{ false, "efi_sys", "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" },
{ false, "free", "Apple_Free" },
{ false, "ddm", "DDM" },
{ false, NULL, "Apple_partition_map" },
@@ -1247,6 +1254,11 @@ STDMETHODIMP CAdcDecoder::Code(ISequentialInStream *inStream,
}
STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
Int32 testMode, IArchiveExtractCallback *extractCallback)
{
@@ -1292,6 +1304,9 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
CAdcDecoder *adcCoderSpec = new CAdcDecoder();
CMyComPtr<ICompressCoder> adcCoder = adcCoderSpec;
NCompress::NLzfse::CDecoder *lzfseCoderSpec = new NCompress::NLzfse::CDecoder();
CMyComPtr<ICompressCoder> lzfseCoder = lzfseCoderSpec;
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(extractCallback, false);
@@ -1419,6 +1434,12 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
opRes = NExtract::NOperationResult::kDataError;
break;
}
case METHOD_LZFSE:
{
res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, progress);
break;
}
default:
opRes = NExtract::NOperationResult::kUnsupportedMethod;
@@ -1490,6 +1511,9 @@ class CInStream:
CAdcDecoder *adcCoderSpec;
CMyComPtr<ICompressCoder> adcCoder;
NCompress::NLzfse::CDecoder *lzfseCoderSpec;
CMyComPtr<ICompressCoder> lzfseCoder;
CBufPtrSeqOutStream *outStreamSpec;
CMyComPtr<ISequentialOutStream> outStream;
@@ -1651,6 +1675,15 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
if (res == S_OK && bzip2CoderSpec->GetInputProcessedSize() != block.PackSize)
res = S_FALSE;
break;
case METHOD_LZFSE:
if (!lzfseCoder)
{
lzfseCoderSpec = new NCompress::NLzfse::CDecoder();
lzfseCoder = lzfseCoderSpec;
}
res = lzfseCoder->Code(inStream, outStream, &block.PackSize, &block.UnpSize, NULL);
break;
default:
return E_FAIL;
@@ -1738,6 +1771,7 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream)
case METHOD_ADC:
case METHOD_ZLIB:
case METHOD_BZIP2:
case METHOD_LZFSE:
case METHOD_END:
break;
default:

4
CPP/7zip/Archive/ExtHandler.cpp
View File

@@ -461,8 +461,8 @@ bool CHeader::Parse(const Byte *p)
if (Is64Bit())
{
HI_32(150, NumBlocks);
// HI_32(154, NumBlocksSuper);
HI_32(0x150, NumBlocks);
// HI_32(0x154, NumBlocksSuper);
HI_32(0x158, NumFreeBlocks);
}

2
CPP/7zip/Archive/Iso/IsoIn.cpp
View File

@@ -180,7 +180,7 @@ UInt32 CInArchive::ReadDigits(int numDigits)
Byte b = ReadByte();
if (b < '0' || b > '9')
{
if (b == 0) // it's bug in some CD's
if (b == 0 || b == ' ') // it's bug in some CD's
b = '0';
else
throw CHeaderErrorException();

8
CPP/7zip/Archive/Nsis/NsisIn.cpp
View File

@@ -2489,7 +2489,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p)
if (c2 == NS_3_CODE_VAR)
// if (c2 <= NS_3_CODE_SKIP && c2 != NS_3_CODE_SHELL && c2 != 1)
{
if ((strData[i+ 2] & 0x80) != 0)
if ((strData[i + 2] & 0x80) != 0)
{
// const char *p2 = (const char *)(strData + i + 1);
// p2 = p2;
@@ -5048,6 +5048,12 @@ HRESULT CInArchive::Parse()
DetectNsisType(bhEntries, _data + bhEntries.Offset);
Decoder.IsNsisDeflate = (NsisType != k_NsisType_Nsis3);
// some NSIS files (that are not detected as k_NsisType_Nsis3)
// use original (non-NSIS) Deflate
// How to detect these cases?
// Decoder.IsNsisDeflate = false;
#ifdef NSIS_SCRIPT

90
CPP/7zip/Archive/NtfsHandler.cpp
View File

@@ -341,15 +341,19 @@ bool CVolInfo::Parse(const Byte *p, unsigned size)
struct CAttr
{
UInt32 Type;
Byte NonResident;
// Non-Resident
Byte CompressionUnit;
// UInt32 Len;
UString2 Name;
// UInt16 Flags;
// UInt16 Instance;
CByteBuffer Data;
Byte NonResident;
// Non-Resident
Byte CompressionUnit;
UInt64 LowVcn;
UInt64 HighVcn;
UInt64 AllocatedSize;
@@ -408,15 +412,16 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
return 8; // required size is 4, but attributes are 8 bytes aligned. So we return 8
if (size < 0x18)
return 0;
PRF(printf(" T=%2X", Type));
UInt32 len = Get32(p + 0x04);
UInt32 len = Get32(p + 4);
PRF(printf(" L=%3d", len));
if (len > size)
return 0;
if ((len & 7) != 0)
return 0;
NonResident = p[0x08];
NonResident = p[8];
{
unsigned nameLen = p[9];
UInt32 nameOffset = Get16(p + 0x0A);
@@ -437,6 +442,7 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
UInt32 dataSize;
UInt32 offs;
if (NonResident)
{
if (len < 0x40)
@@ -472,16 +478,19 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
{
if (len < 0x18)
return 0;
PRF(printf(" RES"));
dataSize = Get32(p + 0x10);
PRF(printf(" dataSize=%3d", dataSize));
offs = Get16(p + 0x14);
G32(p + 0x10, dataSize);
G16(p + 0x14, offs);
// G16(p + 0x16, ResidentFlags);
PRF(printf(" RES"));
PRF(printf(" dataSize=%3d", dataSize));
// PRF(printf(" ResFlags=%4X", ResidentFlags));
}
if (offs > len || dataSize > len || len - dataSize < offs)
return 0;
Data.CopyFrom(p + offs, dataSize);
#ifdef SHOW_DEBUG_INFO
PRF(printf(" : "));
for (unsigned i = 0; i < Data.Size(); i++)
@@ -489,16 +498,19 @@ UInt32 CAttr::Parse(const Byte *p, unsigned size)
PRF(printf(" %02X", (unsigned)Data[i]));
}
#endif
return len;
}
bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax, unsigned compressionUnit) const
{
const Byte *p = Data;
unsigned size = (unsigned)Data.Size();
UInt64 vcn = LowVcn;
UInt64 lcn = 0;
UInt64 highVcn1 = HighVcn + 1;
const UInt64 highVcn1 = HighVcn + 1;
if (LowVcn != extents.Back().Virt || highVcn1 > (UInt64)1 << 63)
return false;
@@ -528,15 +540,30 @@ bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax,
if ((highVcn1 - vcn) < vSize)
return false;
CExtent e;
e.Virt = vcn;
vcn += vSize;
num = (b >> 4) & 0xF;
if (num > 8 || num > size)
return false;
CExtent e;
e.Virt = vcn;
if (num == 0)
{
// Sparse
/* if Unit is compressed, it can have many Elements for each compressed Unit:
and last Element for unit MUST be without LCN.
Element 0: numCompressedClusters2, LCN_0
Element 1: numCompressedClusters2, LCN_1
...
Last Element : (16 - total_clusters_in_previous_elements), no LCN
*/
// sparse is not allowed for (compressionUnit == 0) ? Why ?
if (compressionUnit == 0)
return false;
e.Phy = kEmptyExtent;
}
else
@@ -553,9 +580,10 @@ bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax,
return false;
e.Phy = lcn;
}
extents.Add(e);
vcn += vSize;
}
CExtent e;
e.Phy = kEmptyExtent;
e.Virt = vcn;
@@ -563,10 +591,11 @@ bool CAttr::ParseExtents(CRecordVector<CExtent> &extents, UInt64 numClustersMax,
return (highVcn1 == vcn);
}
static const UInt64 kEmptyTag = (UInt64)(Int64)-1;
static const unsigned kNumCacheChunksLog = 1;
static const size_t kNumCacheChunks = (1 << kNumCacheChunksLog);
static const size_t kNumCacheChunks = (size_t)1 << kNumCacheChunksLog;
class CInStream:
public IInStream,
@@ -734,24 +763,27 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
while (_curRem == 0)
{
UInt64 cacheTag = _virtPos >> _chunkSizeLog;
UInt32 cacheIndex = (UInt32)cacheTag & (kNumCacheChunks - 1);
const UInt64 cacheTag = _virtPos >> _chunkSizeLog;
const size_t cacheIndex = (size_t)cacheTag & (kNumCacheChunks - 1);
if (_tags[cacheIndex] == cacheTag)
{
UInt32 chunkSize = (UInt32)1 << _chunkSizeLog;
UInt32 offset = (UInt32)_virtPos & (chunkSize - 1);
UInt32 cur = MyMin(chunkSize - offset, size);
const size_t chunkSize = (size_t)1 << _chunkSizeLog;
const size_t offset = (size_t)_virtPos & (chunkSize - 1);
size_t cur = chunkSize - offset;
if (cur > size)
cur = size;
memcpy(data, _outBuf + (cacheIndex << _chunkSizeLog) + offset, cur);
*processedSize = cur;
*processedSize = (UInt32)cur;
_virtPos += cur;
return S_OK;
}
PRF2(printf("\nVirtPos = %6d", _virtPos));
UInt32 comprUnitSize = (UInt32)1 << CompressionUnit;
UInt64 virtBlock = _virtPos >> BlockSizeLog;
UInt64 virtBlock2 = virtBlock & ~((UInt64)comprUnitSize - 1);
const UInt32 comprUnitSize = (UInt32)1 << CompressionUnit;
const UInt64 virtBlock = _virtPos >> BlockSizeLog;
const UInt64 virtBlock2 = virtBlock & ~((UInt64)comprUnitSize - 1);
unsigned left = 0, right = Extents.Size();
for (;;)
@@ -766,7 +798,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
}
bool isCompressed = false;
UInt64 virtBlock2End = virtBlock2 + comprUnitSize;
const UInt64 virtBlock2End = virtBlock2 + comprUnitSize;
if (CompressionUnit != 0)
for (unsigned i = left; i < Extents.Size(); i++)
{
@@ -802,7 +834,9 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
_curRem = next - _virtPos;
break;
}
bool thereArePhy = false;
for (unsigned i2 = left; i2 < Extents.Size(); i2++)
{
const CExtent &e = Extents[i2];
@@ -814,6 +848,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
break;
}
}
if (!thereArePhy)
{
_curRem = (Extents[i + 1].Virt << BlockSizeLog) - _virtPos;
@@ -823,6 +858,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
size_t offs = 0;
UInt64 curVirt = virtBlock2;
for (i = left; i < Extents.Size(); i++)
{
const CExtent &e = Extents[i];
@@ -845,6 +881,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
_physPos += compressed;
offs += compressed;
}
size_t destLenMax = GetCuSize();
size_t destLen = destLenMax;
const UInt64 rem = Size - (virtBlock2 << BlockSizeLog);
@@ -863,6 +900,7 @@ STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
return S_FALSE;
}
}
if (size > _curRem)
size = (UInt32)_curRem;
HRESULT res = S_OK;
@@ -913,6 +951,12 @@ static HRESULT DataParseExtents(unsigned clusterSizeLog, const CObjectVector<CAt
const CAttr &attr0 = attrs[attrIndex];
/*
if (attrs[attrIndexLim - 1].HighVcn + 1 != (attr0.AllocatedSize >> clusterSizeLog))
{
}
*/
if (attr0.AllocatedSize < attr0.Size ||
(attrs[attrIndexLim - 1].HighVcn + 1) != (attr0.AllocatedSize >> clusterSizeLog) ||
(attr0.AllocatedSize & ((1 << clusterSizeLog) - 1)) != 0)

320
CPP/7zip/Archive/PeHandler.cpp
View File

@@ -109,7 +109,8 @@ void CVersion::ToProp(NCOM::CPropVariant &prop)
prop = sz;
}
static const unsigned kHeaderSize = 4 + 20;
static const unsigned kCoffHeaderSize = 20;
static const unsigned kPeHeaderSize = 4 + kCoffHeaderSize;
static const unsigned k_OptHeader32_Size_MIN = 96;
static const unsigned k_OptHeader64_Size_MIN = 112;
@@ -125,15 +126,14 @@ struct CHeader
UInt16 OptHeaderSize;
UInt16 Flags;
bool Parse(const Byte *p);
void ParseBase(const Byte *p);
bool ParseCoff(const Byte *p);
bool ParsePe(const Byte *p);
bool IsDll() const { return (Flags & PE_IMAGE_FILE_DLL) != 0; }
};
bool CHeader::Parse(const Byte *p)
void CHeader::ParseBase(const Byte *p)
{
if (Get32(p) != k_Signature32)
return false;
p += 4;
G16( 0, Machine);
G16( 2, NumSections);
G32( 4, Time);
@@ -141,6 +141,13 @@ bool CHeader::Parse(const Byte *p)
G32(12, NumSymbols);
G16(16, OptHeaderSize);
G16(18, Flags);
}
bool CHeader::ParsePe(const Byte *p)
{
if (Get32(p) != k_Signature32)
return false;
ParseBase(p + 4);
return OptHeaderSize >= k_OptHeader32_Size_MIN;
}
@@ -380,6 +387,10 @@ void CSection::Parse(const Byte *p)
G32(36, Flags);
}
// IMAGE_FILE_*
static const CUInt32PCharPair g_HeaderCharacts[] =
{
{ 1, "Executable" },
@@ -399,39 +410,65 @@ static const CUInt32PCharPair g_HeaderCharacts[] =
{ 15, "Big-Endian" }
};
static const CUInt32PCharPair g_DllCharacts[] =
// IMAGE_DLLCHARACTERISTICS_*
static const char * const g_DllCharacts[] =
{
{ 5, "HighEntropyVA" },
{ 6, "Relocated" },
{ 7, "Integrity" },
{ 8, "NX-Compatible" },
{ 9, "NoIsolation" },
{ 10, "NoSEH" },
{ 11, "NoBind" },
{ 12, "AppContainer" },
{ 13, "WDM" },
{ 14, "GuardCF" },
{ 15, "TerminalServerAware" }
NULL
, NULL
, NULL
, NULL
, NULL
, "HighEntropyVA"
, "Relocated"
, "Integrity"
, "NX-Compatible"
, "NoIsolation"
, "NoSEH"
, "NoBind"
, "AppContainer"
, "WDM"
, "GuardCF"
, "TerminalServerAware"
};
static const CUInt32PCharPair g_SectFlags[] =
// IMAGE_SCN_* constants:
static const char * const g_SectFlags[] =
{
{ 3, "NoPad" },
{ 5, "Code" },
{ 6, "InitializedData" },
{ 7, "UninitializedData" },
{ 9, "Comments" },
{ 11, "Remove" },
{ 12, "COMDAT" },
{ 15, "GP" },
{ 24, "ExtendedRelocations" },
{ 25, "Discardable" },
{ 26, "NotCached" },
{ 27, "NotPaged" },
{ 28, "Shared" },
{ 29, "Execute" },
{ 30, "Read" },
{ 31, "Write" }
NULL
, NULL
, NULL
, "NoPad"
, NULL
, "Code"
, "InitializedData"
, "UninitializedData"
, "Other"
, "Comments"
, NULL // OVER
, "Remove"
, "COMDAT"
, NULL
, "NO_DEFER_SPEC_EXC"
, "GP" // MEM_FARDATA
, NULL // SYSHEAP
, "PURGEABLE" // 16BIT
, "LOCKED"
, "PRELOAD"
, NULL
, NULL
, NULL
, NULL
, "ExtendedRelocations"
, "Discardable"
, "NotCached"
, "NotPaged"
, "Shared"
, "Execute"
, "Read"
, "Write"
};
static const CUInt32PCharPair g_MachinePairs[] =
@@ -699,7 +736,6 @@ class CHandler:
{
CMyComPtr<IInStream> _stream;
CObjectVector<CSection> _sections;
UInt32 _peOffset;
CHeader _header;
UInt32 _totalSize;
Int32 _mainSubfile;
@@ -720,9 +756,12 @@ class CHandler:
bool _parseResources;
bool _checksumError;
bool IsOpt() const { return _header.OptHeaderSize != 0; }
COptHeader _optHeader;
bool _allowTail;
bool _coffMode;
HRESULT LoadDebugSections(IInStream *stream, bool &thereIsSection);
HRESULT Open2(IInStream *stream, IArchiveOpenCallback *callback);
@@ -742,7 +781,10 @@ class CHandler:
}
public:
CHandler(): _allowTail(false) {}
CHandler(bool coffMode = false):
_coffMode(coffMode),
_allowTail(coffMode)
{}
MY_UNKNOWN_IMP3(IInArchive, IInArchiveGetStream, IArchiveAllowTail)
INTERFACE_IInArchive(;)
@@ -841,6 +883,34 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
NCOM::CPropVariant prop;
switch (propID)
{
case kpidPhySize: prop = _totalSize; break;
case kpidComment: if (!_versionFullString.IsEmpty()) prop = _versionFullString; break;
case kpidShortComment:
if (!_versionShortString.IsEmpty())
prop = _versionShortString;
else
{
PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop);
}
break;
case kpidName: if (!_originalFilename.IsEmpty()) prop = _originalFilename; break;
// case kpidIsSelfExe: prop = !_header.IsDll(); break;
// case kpidError:
case kpidWarning: if (_checksumError) prop = "Checksum error"; break;
case kpidCpu: PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); break;
case kpidMTime:
case kpidCTime: TimeToProp(_header.Time, prop); break;
case kpidCharacts: FLAGS_TO_PROP(g_HeaderCharacts, _header.Flags, prop); break;
case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break;
default:
if (IsOpt())
switch (propID)
{
case kpidSectAlign: prop = _optHeader.SectAlign; break;
case kpidFileAlign: prop = _optHeader.FileAlign; break;
case kpidLinkerVer:
@@ -857,37 +927,19 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
case kpidInitDataSize: prop = _optHeader.InitDataSize; break;
case kpidUnInitDataSize: prop = _optHeader.UninitDataSize; break;
case kpidImageSize: prop = _optHeader.ImageSize; break;
case kpidPhySize: prop = _totalSize; break;
case kpidHeadersSize: prop = _optHeader.HeadersSize; break;
case kpidChecksum: prop = _optHeader.CheckSum; break;
case kpidComment: if (!_versionFullString.IsEmpty()) prop = _versionFullString; break;
case kpidShortComment:
if (!_versionShortString.IsEmpty())
prop = _versionShortString;
else
{
PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop);
}
break;
case kpidName: if (!_originalFilename.IsEmpty()) prop = _originalFilename; break;
case kpidExtension:
if (_header.IsDll())
prop = _optHeader.IsSybSystem_EFI() ? "efi" : "dll";
prop = "dll";
else if (_optHeader.IsSybSystem_EFI())
prop = "efi";
break;
// case kpidIsSelfExe: prop = !_header.IsDll(); break;
// case kpidError:
case kpidWarning: if (_checksumError) prop = "Checksum error"; break;
case kpidCpu: PAIR_TO_PROP(g_MachinePairs, _header.Machine, prop); break;
case kpidBit64: if (_optHeader.Is64Bit()) prop = true; break;
case kpidSubSystem: TYPE_TO_PROP(g_SubSystems, _optHeader.SubSystem, prop); break;
case kpidMTime:
case kpidCTime: TimeToProp(_header.Time, prop); break;
case kpidCharacts: FLAGS_TO_PROP(g_HeaderCharacts, _header.Flags, prop); break;
case kpidDllCharacts: FLAGS_TO_PROP(g_DllCharacts, _optHeader.DllCharacts, prop); break;
case kpidStackReserve: prop = _optHeader.StackReserve; break;
case kpidStackCommit: prop = _optHeader.StackCommit; break;
@@ -898,8 +950,7 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
// case kpidAddressOfEntryPoint: prop = _optHeader.AddressOfEntryPoint; break;
// case kpidBaseOfCode: prop = _optHeader.BaseOfCode; break;
// case kpidBaseOfData32: if (!_optHeader.Is64Bit()) prop = _optHeader.BaseOfData32; break;
case kpidMainSubfile: if (_mainSubfile >= 0) prop = (UInt32)_mainSubfile; break;
}
}
prop.Detach(value);
return S_OK;
@@ -1056,7 +1107,24 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
case kpidMTime:
case kpidCTime:
TimeToProp(item.IsDebug ? item.Time : _header.Time, prop); break;
case kpidCharacts: if (item.IsRealSect) FLAGS_TO_PROP(g_SectFlags, item.Flags, prop); break;
case kpidCharacts:
if (item.IsRealSect)
{
UInt32 flags = item.Flags;
const UInt32 MY__IMAGE_SCN_ALIGN_MASK = 0x00F00000;
AString s = FlagsToString(g_SectFlags, ARRAY_SIZE(g_SectFlags), item.Flags & ~MY__IMAGE_SCN_ALIGN_MASK);
const UInt32 align = ((flags >> 20) & 0xF);
if (align != 0)
{
char sz[32];
ConvertUInt32ToString(1 << (align - 1), sz);
s.Add_Space();
s += "align_";
s += sz;
}
prop = s;
}
break;
case kpidZerosTailIsAllowed: if (!item.IsRealSect) prop = true; break;
}
}
@@ -1073,8 +1141,17 @@ HRESULT CHandler::LoadDebugSections(IInStream *stream, bool &thereIsSection)
return S_OK;
const unsigned kEntrySize = 28;
UInt32 numItems = debugLink.Size / kEntrySize;
if (numItems * kEntrySize != debugLink.Size || numItems > 16)
if (numItems > 16)
return S_FALSE;
// MAC's EFI file: numItems can be incorrect. Only first CDebugEntry entry is correct.
// debugLink.Size = kEntrySize + some_data, pointed by entry[0].
if (numItems * kEntrySize != debugLink.Size)
{
// return S_FALSE;
if (numItems > 1)
numItems = 1;
}
UInt64 pa = 0;
unsigned i;
@@ -2114,6 +2191,26 @@ HRESULT CHandler::OpenResources(unsigned sectionIndex, IInStream *stream, IArchi
return S_OK;
}
bool CHeader::ParseCoff(const Byte *p)
{
ParseBase(p);
if (PointerToSymbolTable < kCoffHeaderSize)
return false;
if (NumSymbols >= (1 << 24))
return false;
if (OptHeaderSize != 0 && OptHeaderSize < k_OptHeader32_Size_MIN)
return false;
for (unsigned i = 0; i < ARRAY_SIZE(g_MachinePairs); i++)
if (Machine == g_MachinePairs[i].Value)
return true;
if (Machine == 0)
return true;
return false;
}
static inline bool CheckPeOffset(UInt32 pe)
{
// ((pe & 7) == 0) is for most PE files. But there is unusual EFI-PE file that uses unaligned pe value.
@@ -2133,10 +2230,10 @@ API_FUNC_static_IsArc IsArc_Pe(const Byte *p, size_t size)
UInt32 pe = Get32(p + 0x3C);
if (!CheckPeOffset(pe))
return k_IsArc_Res_NO;
if (pe + kHeaderSize > size)
if (pe + kPeHeaderSize > size)
return k_IsArc_Res_NEED_MORE;
CHeader header;
if (!header.Parse(p + pe))
if (!header.ParsePe(p + pe))
return k_IsArc_Res_NO;
return k_IsArc_Res_YES;
}
@@ -2144,32 +2241,47 @@ API_FUNC_static_IsArc IsArc_Pe(const Byte *p, size_t size)
HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
{
UInt32 coffOffset = 0;
if (_coffMode)
{
Byte h[kStartSize];
_mainSubfile = -1;
RINOK(ReadStream_FALSE(stream, h, kStartSize));
if (h[0] != 'M' || h[1] != 'Z')
return S_FALSE;
/* most of PE files contain 0x0090 at offset 2.
But some rare PE files contain another values. So we don't use that check.
if (Get16(h + 2) != 0x90) return false; */
_peOffset = Get32(h + 0x3C);
if (!CheckPeOffset(_peOffset))
Byte h[kCoffHeaderSize];
RINOK(ReadStream_FALSE(stream, h, kCoffHeaderSize));
if (!_header.ParseCoff(h))
return S_FALSE;
}
else
{
Byte h[kHeaderSize];
RINOK(stream->Seek(_peOffset, STREAM_SEEK_SET, NULL));
RINOK(ReadStream_FALSE(stream, h, kHeaderSize));
if (!_header.Parse(h))
return S_FALSE;
UInt32 _peOffset;
{
Byte h[kStartSize];
RINOK(ReadStream_FALSE(stream, h, kStartSize));
if (h[0] != 'M' || h[1] != 'Z')
return S_FALSE;
/* most of PE files contain 0x0090 at offset 2.
But some rare PE files contain another values. So we don't use that check.
if (Get16(h + 2) != 0x90) return false; */
_peOffset = Get32(h + 0x3C);
if (!CheckPeOffset(_peOffset))
return S_FALSE;
coffOffset = _peOffset + 4;
}
{
Byte h[kPeHeaderSize];
RINOK(stream->Seek(_peOffset, STREAM_SEEK_SET, NULL));
RINOK(ReadStream_FALSE(stream, h, kPeHeaderSize));
if (!_header.ParsePe(h))
return S_FALSE;
}
}
UInt32 bufSize = _header.OptHeaderSize + (UInt32)_header.NumSections * kSectionSize;
_totalSize = _peOffset + kHeaderSize + bufSize;
const UInt32 optStart = coffOffset + kCoffHeaderSize;
const UInt32 bufSize = _header.OptHeaderSize + (UInt32)_header.NumSections * kSectionSize;
_totalSize = optStart + bufSize;
CByteBuffer buffer(bufSize);
RINOK(ReadStream_FALSE(stream, buffer, bufSize));
if (_header.OptHeaderSize != 0)
if (!_optHeader.Parse(buffer, _header.OptHeaderSize))
return S_FALSE;
@@ -2207,7 +2319,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
for (i = 0; i < _sections.Size(); i++)
_sections[i].UpdateTotalSize(_totalSize);
bool thereISDebug;
bool thereISDebug = false;
if (IsOpt())
{
RINOK(LoadDebugSections(stream, thereISDebug));
const CDirLink &certLink = _optHeader.DirItems[kDirLink_Certificate];
@@ -2256,8 +2370,9 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
_totalSize += (UInt32)k;
}
}
}
if (_header.NumSymbols > 0 && _header.PointerToSymbolTable >= 512)
if (_header.NumSymbols > 0 && _header.PointerToSymbolTable >= optStart)
{
if (_header.NumSymbols >= (1 << 24))
return S_FALSE;
@@ -2306,11 +2421,12 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
}
if (IsOpt())
if (_optHeader.CheckSum != 0)
{
RINOK(stream->Seek(0, STREAM_SEEK_SET, NULL));
UInt32 checkSum = 0;
RINOK(CalcCheckSum(stream, _totalSize, _peOffset + kHeaderSize + k_CheckSum_Field_Offset, checkSum));
RINOK(CalcCheckSum(stream, _totalSize, optStart + k_CheckSum_Field_Offset, checkSum));
_checksumError = (checkSum != _optHeader.CheckSum);
}
@@ -2333,6 +2449,7 @@ HRESULT CHandler::Open2(IInStream *stream, IArchiveOpenCallback *callback)
const CSection &sect = _sections[i];
CMixItem mixItem;
mixItem.SectionIndex = i;
if (IsOpt())
if (_parseResources && sect.Name == ".rsrc" && _items.IsEmpty())
{
const unsigned numMixItems = _mixItems.Size();
@@ -2480,6 +2597,8 @@ STDMETHODIMP CHandler::Close()
{
_totalSize = 0;
_checksumError = false;
_mainSubfile = -1;
_stream.Release();
_sections.Clear();
_mixItems.Clear();
@@ -2675,10 +2794,41 @@ REGISTER_ARC_I(
0,
NArcInfoFlags::kPreArc,
IsArc_Pe)
}
namespace NCoff {
API_FUNC_static_IsArc IsArc_Coff(const Byte *p, size_t size)
{
if (size < NPe::kCoffHeaderSize)
return k_IsArc_Res_NEED_MORE;
NPe::CHeader header;
if (!header.ParseCoff(p))
return k_IsArc_Res_NO;
return k_IsArc_Res_YES;
}
}
/*
static const Byte k_Signature[] =
{
2, 0x4C, 0x01, // x86
2, 0x64, 0x86, // x64
2, 0x64, 0xAA // ARM64
};
REGISTER_ARC_I_CLS(
*/
REGISTER_ARC_I_CLS_NO_SIG(
NPe::CHandler(true),
"COFF", "obj", 0, 0xC6,
// k_Signature,
0,
// NArcInfoFlags::kMultiSignature |
NArcInfoFlags::kStartOpen,
IsArc_Coff)
}
namespace NTe {

2
CPP/7zip/Archive/Rar/Rar5Handler.cpp
View File

@@ -1363,7 +1363,7 @@ static const Byte kProps[] =
kpidCharacts,
kpidSymLink,
kpidHardLink,
kpidCopyLink,
kpidCopyLink
};

5
CPP/7zip/Archive/SquashfsHandler.cpp
View File

@@ -60,6 +60,7 @@ UInt64 Get64b(const Byte *p, bool be) { return be ? GetBe64(p) : GetUi64(p); }
static const UInt32 kSignature32_LE = 0x73717368;
static const UInt32 kSignature32_BE = 0x68737173;
static const UInt32 kSignature32_LZ = 0x71736873;
static const UInt32 kSignature32_B2 = 0x73687371;
#define kMethod_ZLIB 1
#define kMethod_LZMA 2
@@ -225,6 +226,7 @@ struct CHeader
case kSignature32_LE: break;
case kSignature32_BE: be = true; break;
case kSignature32_LZ: SeveralMethods = true; break;
case kSignature32_B2: SeveralMethods = true; be = true; break;
default: return false;
}
GET_32 (4, NumInodes);
@@ -2258,7 +2260,8 @@ STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream)
static const Byte k_Signature[] = {
4, 'h', 's', 'q', 's',
4, 's', 'q', 's', 'h',
4, 's', 'h', 's', 'q' };
4, 's', 'h', 's', 'q',
4, 'q', 's', 'h', 's' };
REGISTER_ARC_I(
"SquashFS", "squashfs", 0, 0xD2,

17
CPP/7zip/Archive/Tar/TarIn.cpp
View File

@@ -26,24 +26,25 @@ static void MyStrNCpy(char *dest, const char *src, unsigned size)
}
}
static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res)
static bool OctalToNumber(const char *srcString, unsigned size, UInt64 &res, bool allowEmpty = false)
{
res = 0;
char sz[32];
MyStrNCpy(sz, srcString, size);
sz[size] = 0;
const char *end;
unsigned i;
for (i = 0; sz[i] == ' '; i++);
if (sz[i] == 0)
return allowEmpty;
res = ConvertOctStringToUInt64(sz + i, &end);
if (end == sz + i)
return false;
return (*end == ' ' || *end == 0);
}
static bool OctalToNumber32(const char *srcString, unsigned size, UInt32 &res)
static bool OctalToNumber32(const char *srcString, unsigned size, UInt32 &res, bool allowEmpty = false)
{
UInt64 res64;
if (!OctalToNumber(srcString, size, res64))
if (!OctalToNumber(srcString, size, res64, allowEmpty))
return false;
res = (UInt32)res64;
return (res64 <= 0xFFFFFFFF);
@@ -123,7 +124,8 @@ API_FUNC_IsArc IsArc_Tar(const Byte *p2, size_t size)
p += NFileHeader::kNameSize;
UInt32 mode;
CHECK(OctalToNumber32(p, 8, mode)); p += 8;
// we allow empty Mode value for LongName prefix items
CHECK(OctalToNumber32(p, 8, mode, true)); p += 8;
// if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0;
p += 8;
@@ -194,7 +196,8 @@ static HRESULT GetNextItemReal(ISequentialInStream *stream, bool &filled, CItemE
(item.Name.Len() == NFileHeader::kNameSize ||
item.Name.Len() == NFileHeader::kNameSize - 1);
RIF(OctalToNumber32(p, 8, item.Mode)); p += 8;
// we allow empty Mode value for LongName prefix items
RIF(OctalToNumber32(p, 8, item.Mode, true)); p += 8;
if (!OctalToNumber32(p, 8, item.UID)) item.UID = 0; p += 8;
if (!OctalToNumber32(p, 8, item.GID)) item.GID = 0; p += 8;

659
CPP/7zip/Archive/XzHandler.cpp
View File

@@ -7,8 +7,11 @@
#include "../../Common/ComTry.h"
#include "../../Common/Defs.h"
#include "../../Common/IntToString.h"
#include "../../Common/MyBuffer.h"
#include "../../Common/StringToInt.h"
#include "../../Windows/PropVariant.h"
#include "../../Windows/System.h"
#include "../Common/CWrappers.h"
#include "../Common/ProgressUtils.h"
@@ -33,9 +36,19 @@ namespace NXz {
#define k_LZMA2_Name "LZMA2"
struct CBlockInfo
{
unsigned StreamFlags;
UInt64 PackPos;
UInt64 PackSize; // pure value from Index record, it doesn't include pad zeros
UInt64 UnpackPos;
};
class CHandler:
public IInArchive,
public IArchiveOpenSeq,
public IInArchiveGetStream,
#ifndef EXTRACT_ONLY
public IOutArchive,
public ISetProperties,
@@ -48,9 +61,7 @@ class CHandler:
bool _isArc;
bool _needSeekToStart;
bool _phySize_Defined;
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
bool _firstBlockWasRead;
AString _methodsString;
@@ -58,8 +69,20 @@ class CHandler:
UInt32 _filterId;
UInt64 _numSolidBytes;
HRESULT SetSolidFromString(const UString &s);
HRESULT SetSolidFromPROPVARIANT(const PROPVARIANT &value);
HRESULT SetProperty(const wchar_t *name, const PROPVARIANT &value);
void InitSolid()
{
_numSolidBytes = XZ_PROPS__BLOCK_SIZE__AUTO;
}
void Init()
{
InitSolid();
_filterId = 0;
CMultiMethodProps::Init();
}
@@ -83,6 +106,7 @@ class CHandler:
public:
MY_QUERYINTERFACE_BEGIN2(IInArchive)
MY_QUERYINTERFACE_ENTRY(IArchiveOpenSeq)
MY_QUERYINTERFACE_ENTRY(IInArchiveGetStream)
#ifndef EXTRACT_ONLY
MY_QUERYINTERFACE_ENTRY(IOutArchive)
MY_QUERYINTERFACE_ENTRY(ISetProperties)
@@ -92,22 +116,45 @@ public:
INTERFACE_IInArchive(;)
STDMETHOD(OpenSeq)(ISequentialInStream *stream);
STDMETHOD(GetStream)(UInt32 index, ISequentialInStream **stream);
#ifndef EXTRACT_ONLY
INTERFACE_IOutArchive(;)
STDMETHOD(SetProperties)(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps);
#endif
size_t _blocksArraySize;
CBlockInfo *_blocks;
UInt64 _maxBlocksSize;
CMyComPtr<IInStream> _stream;
CMyComPtr<ISequentialInStream> _seqStream;
CXzBlock _firstBlock;
CHandler();
~CHandler();
HRESULT SeekToPackPos(UInt64 pos)
{
return _stream->Seek(pos, STREAM_SEEK_SET, NULL);
}
};
CHandler::CHandler()
CHandler::CHandler():
_blocks(NULL),
_blocksArraySize(0)
{
#ifndef EXTRACT_ONLY
Init();
#endif
}
CHandler::~CHandler()
{
MyFree(_blocks);
}
static const Byte kProps[] =
{
@@ -120,7 +167,9 @@ static const Byte kArcProps[] =
{
kpidMethod,
kpidNumStreams,
kpidNumBlocks
kpidNumBlocks,
kpidClusterSize,
kpidCharacts
};
IMP_IInArchive_Props
@@ -177,7 +226,7 @@ static const CMethodNamePair g_NamePairs[] =
{ XZ_ID_LZMA2, "LZMA2" }
};
static AString GetMethodString(const CXzFilter &f)
static void AddMethodString(AString &s, const CXzFilter &f)
{
const char *p = NULL;
for (unsigned i = 0; i < ARRAY_SIZE(g_NamePairs); i++)
@@ -193,7 +242,7 @@ static AString GetMethodString(const CXzFilter &f)
p = temp;
}
AString s (p);
s += p;
if (f.propsSize > 0)
{
@@ -210,13 +259,6 @@ static AString GetMethodString(const CXzFilter &f)
s += ']';
}
}
return s;
}
static void AddString(AString &dest, const AString &src)
{
dest.Add_Space_if_NotEmpty();
dest += src;
}
static const char * const kChecks[] =
@@ -239,27 +281,24 @@ static const char * const kChecks[] =
, NULL
};
static AString GetCheckString(const CXzs &xzs)
static void AddCheckString(AString &s, const CXzs &xzs)
{
size_t i;
UInt32 mask = 0;
for (i = 0; i < xzs.num; i++)
mask |= ((UInt32)1 << XzFlags_GetCheckType(xzs.streams[i].flags));
AString s;
for (i = 0; i <= XZ_CHECK_MASK; i++)
if (((mask >> i) & 1) != 0)
{
AString s2;
s.Add_Space_if_NotEmpty();
if (kChecks[i])
s2 = kChecks[i];
s += kChecks[i];
else
{
s2 = "Check-";
s2.Add_UInt32((UInt32)i);
s += "Check-";
s.Add_UInt32((UInt32)i);
}
AddString(s, s2);
}
return s;
}
STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
@@ -272,11 +311,26 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
case kpidNumStreams: if (_stat.NumStreams_Defined) prop = _stat.NumStreams; break;
case kpidNumBlocks: if (_stat.NumBlocks_Defined) prop = _stat.NumBlocks; break;
case kpidUnpackSize: if (_stat.UnpackSize_Defined) prop = _stat.OutSize; break;
case kpidClusterSize: if (_stat.NumBlocks_Defined && _stat.NumBlocks > 1) prop = _maxBlocksSize; break;
case kpidCharacts:
if (_firstBlockWasRead)
{
AString s;
if (XzBlock_HasPackSize(&_firstBlock))
s.Add_OptSpaced("BlockPackSize");
if (XzBlock_HasUnpackSize(&_firstBlock))
s.Add_OptSpaced("BlockUnpackSize");
if (!s.IsEmpty())
prop = s;
}
break;
case kpidMethod: if (!_methodsString.IsEmpty()) prop = _methodsString; break;
case kpidErrorFlags:
{
UInt32 v = 0;
if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;;
if (!_isArc) v |= kpv_ErrorFlags_IsNotArc;
if (_stat.UnexpectedEnd) v |= kpv_ErrorFlags_UnexpectedEnd;
if (_stat.DataAfterEnd) v |= kpv_ErrorFlags_DataAfterEnd;
if (_stat.HeadersError) v |= kpv_ErrorFlags_HeadersError;
@@ -284,6 +338,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
if (_stat.DataError) v |= kpv_ErrorFlags_DataError;
if (_stat.CrcError) v |= kpv_ErrorFlags_CrcError;
prop = v;
break;
}
case kpidMainSubfile:
{
// if (_blocks) prop = (UInt32)0;
break;
}
}
prop.Detach(value);
@@ -409,9 +470,15 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal
SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.vt, &isIndex, &headerSizeRes);
if (res2 == SZ_OK && !isIndex)
{
_firstBlockWasRead = true;
_firstBlock = block;
unsigned numFilters = XzBlock_GetNumFilters(&block);
for (unsigned i = 0; i < numFilters; i++)
AddString(_methodsString, GetMethodString(block.filters[i]));
{
_methodsString.Add_Space_if_NotEmpty();
AddMethodString(_methodsString, block.filters[i]);
}
}
}
}
@@ -422,7 +489,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal
RINOK(callback->SetTotal(NULL, &_stat.PhySize));
}
CSeekInStreamWrap inStreamImp;;
CSeekInStreamWrap inStreamImp;
inStreamImp.Init(inStream);
@@ -460,7 +527,64 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal
_stat.NumBlocks = Xzs_GetNumBlocks(&xzs.p);
_stat.NumBlocks_Defined = true;
AddString(_methodsString, GetCheckString(xzs.p));
AddCheckString(_methodsString, xzs.p);
const size_t numBlocks = (size_t)_stat.NumBlocks + 1;
const size_t bytesAlloc = numBlocks * sizeof(CBlockInfo);
if (bytesAlloc / sizeof(CBlockInfo) == _stat.NumBlocks + 1)
{
_blocks = (CBlockInfo *)MyAlloc(bytesAlloc);
if (_blocks)
{
unsigned blockIndex = 0;
UInt64 unpackPos = 0;
for (size_t si = xzs.p.num; si != 0;)
{
si--;
const CXzStream &str = xzs.p.streams[si];
UInt64 packPos = str.startOffset + XZ_STREAM_HEADER_SIZE;
for (size_t bi = 0; bi < str.numBlocks; bi++)
{
const CXzBlockSizes &bs = str.blocks[bi];
const UInt64 packSizeAligned = bs.totalSize + ((0 - (unsigned)bs.totalSize) & 3);
if (bs.unpackSize != 0)
{
if (blockIndex >= _stat.NumBlocks)
return E_FAIL;
CBlockInfo &block = _blocks[blockIndex++];
block.StreamFlags = str.flags;
block.PackSize = bs.totalSize; // packSizeAligned;
block.PackPos = packPos;
block.UnpackPos = unpackPos;
}
packPos += packSizeAligned;
unpackPos += bs.unpackSize;
if (_maxBlocksSize < bs.unpackSize)
_maxBlocksSize = bs.unpackSize;
}
}
/*
if (blockIndex != _stat.NumBlocks)
{
// there are Empty blocks;
}
*/
if (_stat.OutSize != unpackPos)
return E_FAIL;
CBlockInfo &block = _blocks[blockIndex++];
block.StreamFlags = 0;
block.PackSize = 0;
block.PackPos = 0;
block.UnpackPos = unpackPos;
_blocksArraySize = blockIndex;
}
}
}
else
{
@@ -474,6 +598,8 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal
return S_OK;
}
STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *callback)
{
COM_TRY_BEGIN
@@ -499,39 +625,297 @@ STDMETHODIMP CHandler::Close()
_isArc = false;
_needSeekToStart = false;
_phySize_Defined = false;
_firstBlockWasRead = false;
_methodsString.Empty();
_stream.Release();
_seqStream.Release();
MyFree(_blocks);
_blocks = NULL;
_blocksArraySize = 0;
_maxBlocksSize = 0;
return S_OK;
}
struct CXzUnpackerCPP2
{
Byte *InBuf;
// Byte *OutBuf;
CXzUnpacker p;
CXzUnpackerCPP2();
~CXzUnpackerCPP2();
};
CXzUnpackerCPP2::CXzUnpackerCPP2(): InBuf(NULL)
// , OutBuf(NULL)
{
XzUnpacker_Construct(&p, &g_Alloc);
}
CXzUnpackerCPP2::~CXzUnpackerCPP2()
{
XzUnpacker_Free(&p);
MidFree(InBuf);
// MidFree(OutBuf);
}
class CInStream:
public IInStream,
public CMyUnknownImp
{
public:
UInt64 _virtPos;
UInt64 Size;
UInt64 _cacheStartPos;
size_t _cacheSize;
CByteBuffer _cache;
// UInt64 _startPos;
CXzUnpackerCPP2 xz;
void InitAndSeek()
{
_virtPos = 0;
_cacheStartPos = 0;
_cacheSize = 0;
// _startPos = startPos;
}
CHandler *_handlerSpec;
CMyComPtr<IUnknown> _handler;
MY_UNKNOWN_IMP1(IInStream)
STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition);
~CInStream();
};
CInStream::~CInStream()
{
// _cache.Free();
}
size_t FindBlock(const CBlockInfo *blocks, size_t numBlocks, UInt64 pos)
{
size_t left = 0, right = numBlocks;
for (;;)
{
size_t mid = (left + right) / 2;
if (mid == left)
return left;
if (pos < blocks[mid].UnpackPos)
right = mid;
else
left = mid;
}
}
static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu,
ISequentialInStream *seqInStream,
unsigned streamFlags,
UInt64 packSize, // pure size from Index record, it doesn't include pad zeros
size_t unpackSize, Byte *dest
// , ICompressProgressInfo *progress
)
{
const size_t kInBufSize = (size_t)1 << 16;
XzUnpacker_Init(&xzu.p);
if (!xzu.InBuf)
{
xzu.InBuf = (Byte *)MidAlloc(kInBufSize);
if (!xzu.InBuf)
return E_OUTOFMEMORY;
}
xzu.p.streamFlags = (UInt16)streamFlags;
XzUnpacker_PrepareToRandomBlockDecoding(&xzu.p);
const UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3);
UInt64 packRem = packSizeAligned;
UInt32 inSize = 0;
SizeT inPos = 0;
SizeT outPos = 0;
HRESULT readRes = S_OK;
for (;;)
{
if (inPos == inSize && readRes == S_OK)
{
inPos = 0;
inSize = 0;
UInt32 rem = kInBufSize;
if (rem > packRem)
rem = (UInt32)packRem;
if (rem != 0)
readRes = seqInStream->Read(xzu.InBuf, rem, &inSize);
}
SizeT inLen = inSize - inPos;
SizeT outLen = unpackSize - outPos;
ECoderStatus status;
SRes res = XzUnpacker_Code(&xzu.p,
dest + outPos, &outLen,
xzu.InBuf + inPos, &inLen,
CODER_FINISH_END, &status);
// return E_OUTOFMEMORY;
// res = SZ_ERROR_CRC;
if (res != SZ_OK)
{
if (res == SZ_ERROR_CRC)
return S_FALSE;
return SResToHRESULT(res);
}
inPos += inLen;
outPos += outLen;
packRem -= inLen;
Bool blockFinished = XzUnpacker_IsBlockFinished(&xzu.p);
if ((inLen == 0 && outLen == 0) || blockFinished)
{
if (packRem != 0 || !blockFinished || unpackSize != outPos)
return S_FALSE;
if (XzUnpacker_GetPackSizeForIndex(&xzu.p) != packSize)
return S_FALSE;
return S_OK;
}
}
}
STDMETHODIMP CInStream::Read(void *data, UInt32 size, UInt32 *processedSize)
{
COM_TRY_BEGIN
if (processedSize)
*processedSize = 0;
if (size == 0)
return S_OK;
{
if (_virtPos >= Size)
return S_OK; // (Size == _virtPos) ? S_OK: E_FAIL;
{
UInt64 rem = Size - _virtPos;
if (size > rem)
size = (UInt32)rem;
}
}
if (size == 0)
return S_OK;
if (_virtPos < _cacheStartPos || _virtPos >= _cacheStartPos + _cacheSize)
{
size_t bi = FindBlock(_handlerSpec->_blocks, _handlerSpec->_blocksArraySize, _virtPos);
const CBlockInfo &block = _handlerSpec->_blocks[bi];
const UInt64 unpackSize = _handlerSpec->_blocks[bi + 1].UnpackPos - block.UnpackPos;
if (_cache.Size() < unpackSize)
return E_FAIL;
_cacheSize = 0;
RINOK(_handlerSpec->SeekToPackPos(block.PackPos));
RINOK(DecodeBlock(xz, _handlerSpec->_seqStream, block.StreamFlags, block.PackSize,
(size_t)unpackSize, _cache));
_cacheStartPos = block.UnpackPos;
_cacheSize = (size_t)unpackSize;
}
{
size_t offset = (size_t)(_virtPos - _cacheStartPos);
size_t rem = _cacheSize - offset;
if (size > rem)
size = (UInt32)rem;
memcpy(data, _cache + offset, size);
_virtPos += size;
if (processedSize)
*processedSize = size;
return S_OK;
}
COM_TRY_END
}
STDMETHODIMP CInStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)
{
switch (seekOrigin)
{
case STREAM_SEEK_SET: break;
case STREAM_SEEK_CUR: offset += _virtPos; break;
case STREAM_SEEK_END: offset += Size; break;
default: return STG_E_INVALIDFUNCTION;
}
if (offset < 0)
return HRESULT_WIN32_ERROR_NEGATIVE_SEEK;
_virtPos = offset;
if (newPosition)
*newPosition = offset;
return S_OK;
}
class CSeekToSeqStream:
public IInStream,
public CMyUnknownImp
{
public:
CMyComPtr<ISequentialInStream> Stream;
MY_UNKNOWN_IMP1(IInStream)
static const UInt64 kMaxBlockSize_for_GetStream = (UInt64)1 << 40;
STDMETHOD(Read)(void *data, UInt32 size, UInt32 *processedSize);
STDMETHOD(Seek)(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition);
};
STDMETHODIMP CSeekToSeqStream::Read(void *data, UInt32 size, UInt32 *processedSize)
STDMETHODIMP CHandler::GetStream(UInt32 index, ISequentialInStream **stream)
{
return Stream->Read(data, size, processedSize);
COM_TRY_BEGIN
*stream = NULL;
if (index != 0)
return E_INVALIDARG;
if (!_stat.UnpackSize_Defined
|| _maxBlocksSize > kMaxBlockSize_for_GetStream
|| _maxBlocksSize != (size_t)_maxBlocksSize)
return S_FALSE;
UInt64 physSize = (UInt64)(sizeof(size_t)) << 29;
bool ramSize_Defined = NSystem::GetRamSize(physSize);
if (ramSize_Defined)
{
if (_maxBlocksSize > physSize / 4)
return S_FALSE;
}
CInStream *spec = new CInStream;
CMyComPtr<ISequentialInStream> specStream = spec;
spec->_cache.Alloc((size_t)_maxBlocksSize);
spec->_handlerSpec = this;
spec->_handler = (IInArchive *)this;
spec->Size = _stat.OutSize;
spec->InitAndSeek();
*stream = specStream.Detach();
return S_OK;
COM_TRY_END
}
STDMETHODIMP CSeekToSeqStream::Seek(Int64, UInt32, UInt64 *) { return E_NOTIMPL; }
@@ -584,6 +968,8 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems,
COM_TRY_END
}
#ifndef EXTRACT_ONLY
STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType)
@@ -592,16 +978,15 @@ STDMETHODIMP CHandler::GetFileTimeType(UInt32 *timeType)
return S_OK;
}
STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems,
IArchiveUpdateCallback *updateCallback)
{
COM_TRY_BEGIN
if (numItems == 0)
{
CSeqOutStreamWrap seqOutStream;
seqOutStream.Init(outStream);
SRes res = Xz_EncodeEmpty(&seqOutStream.vt);
return SResToHRESULT(res);
@@ -641,82 +1026,76 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
NCompress::NXz::CEncoder *encoderSpec = new NCompress::NXz::CEncoder;
CMyComPtr<ICompressCoder> encoder = encoderSpec;
CLzma2EncProps &lzma2Props = encoderSpec->_lzma2Props;
CXzProps &xzProps = encoderSpec->xzProps;
CLzma2EncProps &lzma2Props = xzProps.lzma2Props;
lzma2Props.lzmaProps.level = GetLevel();
CMyComPtr<ISequentialInStream> fileInStream;
RINOK(updateCallback->GetStream(0, &fileInStream));
xzProps.reduceSize = size;
/*
{
NCOM::CPropVariant prop = (UInt64)size;
RINOK(encoderSpec->SetCoderProp(NCoderPropID::kReduceSize, prop));
}
*/
#ifndef _7ZIP_ST
xzProps.numTotalThreads = _numThreads;
#endif
xzProps.blockSize = _numSolidBytes;
if (_numSolidBytes == XZ_PROPS__BLOCK_SIZE__SOLID)
{
xzProps.lzma2Props.blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID;
}
RINOK(encoderSpec->SetCheckSize(_crcSize));
{
CXzFilterProps &filter = xzProps.filterProps;
if (_filterId == XZ_ID_Delta)
{
bool deltaDefined = false;
FOR_VECTOR (j, _filterMethod.Props)
{
const CProp &prop = _filterMethod.Props[j];
if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4)
{
UInt32 delta = (UInt32)prop.Value.ulVal;
if (delta < 1 || delta > 256)
return E_INVALIDARG;
filter.delta = delta;
deltaDefined = true;
}
else
return E_INVALIDARG;
}
if (!deltaDefined)
return E_INVALIDARG;
}
filter.id = _filterId;
}
FOR_VECTOR (i, _methods)
{
COneMethodInfo &m = _methods[i];
/*
SetGlobalLevelTo(m);
#ifndef _7ZIP_ST
CMultiMethodProps::SetMethodThreads(m, _numThreads);
#endif
*/
FOR_VECTOR (j, m.Props)
{
FOR_VECTOR (j, m.Props)
{
const CProp &prop = m.Props[j];
RINOK(encoderSpec->SetCoderProp(prop.Id, prop.Value));
}
const CProp &prop = m.Props[j];
RINOK(encoderSpec->SetCoderProp(prop.Id, prop.Value));
}
}
#ifndef _7ZIP_ST
lzma2Props.numTotalThreads = _numThreads;
#endif
CMyComPtr<ISequentialInStream> fileInStream;
RINOK(updateCallback->GetStream(0, &fileInStream));
CLocalProgress *lps = new CLocalProgress;
CMyComPtr<ICompressProgressInfo> progress = lps;
lps->Init(updateCallback, true);
CXzProps &xzProps = encoderSpec->xzProps;
CXzFilterProps &filter = encoderSpec->filter;
XzProps_Init(&xzProps);
XzFilterProps_Init(&filter);
xzProps.lzma2Props = &lzma2Props;
xzProps.filterProps = (_filterId != 0 ? &filter : NULL);
switch (_crcSize)
{
case 0: xzProps.checkId = XZ_CHECK_NO; break;
case 4: xzProps.checkId = XZ_CHECK_CRC32; break;
case 8: xzProps.checkId = XZ_CHECK_CRC64; break;
case 32: xzProps.checkId = XZ_CHECK_SHA256; break;
default: return E_INVALIDARG;
}
filter.id = _filterId;
if (_filterId == XZ_ID_Delta)
{
bool deltaDefined = false;
FOR_VECTOR (j, _filterMethod.Props)
{
const CProp &prop = _filterMethod.Props[j];
if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4)
{
UInt32 delta = (UInt32)prop.Value.ulVal;
if (delta < 1 || delta > 256)
return E_INVALIDARG;
filter.delta = delta;
deltaDefined = true;
}
}
if (!deltaDefined)
return E_INVALIDARG;
}
return encoderSpec->Code(fileInStream, outStream, NULL, NULL, progress);
}
@@ -747,11 +1126,83 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
}
HRESULT CHandler::SetSolidFromString(const UString &s)
{
UString s2 = s;
s2.MakeLower_Ascii();
{
const wchar_t *start = ((const wchar_t *)s2);
const wchar_t *end;
UInt64 v = ConvertStringToUInt64(start, &end);
if (start == end)
return E_INVALIDARG;
if ((unsigned)(end - start) + 1 != s2.Len())
return E_INVALIDARG;
wchar_t c = *end;
{
unsigned numBits;
switch (c)
{
case 'b': numBits = 0; break;
case 'k': numBits = 10; break;
case 'm': numBits = 20; break;
case 'g': numBits = 30; break;
case 't': numBits = 40; break;
default: return E_INVALIDARG;
}
_numSolidBytes = (v << numBits);
}
}
return S_OK;
}
HRESULT CHandler::SetSolidFromPROPVARIANT(const PROPVARIANT &value)
{
bool isSolid;
switch (value.vt)
{
case VT_EMPTY: isSolid = true; break;
case VT_BOOL: isSolid = (value.boolVal != VARIANT_FALSE); break;
case VT_BSTR:
if (StringToBool(value.bstrVal, isSolid))
break;
return SetSolidFromString(value.bstrVal);
default: return E_INVALIDARG;
}
_numSolidBytes = (isSolid ? XZ_PROPS__BLOCK_SIZE__SOLID : XZ_PROPS__BLOCK_SIZE__AUTO);
return S_OK;
}
HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value)
{
UString name = nameSpec;
name.MakeLower_Ascii();
if (name.IsEmpty())
return E_INVALIDARG;
if (name[0] == L's')
{
name.Delete(0);
if (name.IsEmpty())
return SetSolidFromPROPVARIANT(value);
if (value.vt != VT_EMPTY)
return E_INVALIDARG;
return SetSolidFromString(name);
}
return CMultiMethodProps::SetProperty(name, value);
}
STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)
{
COM_TRY_BEGIN
Init();
for (UInt32 i = 0; i < numProps; i++)
{
RINOK(SetProperty(names[i], values[i]));
@@ -781,7 +1232,9 @@ STDMETHODIMP CHandler::SetProperties(const wchar_t * const *names, const PROPVAR
AString &methodName = _methods[0].MethodName;
if (methodName.IsEmpty())
methodName = k_LZMA2_Name;
else if (!methodName.IsEqualTo_Ascii_NoCase(k_LZMA2_Name))
else if (
!methodName.IsEqualTo_Ascii_NoCase(k_LZMA2_Name)
&& !methodName.IsEqualTo_Ascii_NoCase("xz"))
return E_INVALIDARG;
}

94
CPP/7zip/Archive/Zip/ZipAddCommon.cpp
View File

@@ -36,6 +36,7 @@ static const UInt32 kLzmaHeaderSize = 4 + kLzmaPropsSize;
class CLzmaEncoder:
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressSetCoderPropertiesOpt,
public CMyUnknownImp
{
public:
@@ -46,8 +47,11 @@ public:
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
MY_UNKNOWN_IMP1(ICompressSetCoderProperties)
MY_UNKNOWN_IMP2(
ICompressSetCoderProperties,
ICompressSetCoderPropertiesOpt)
};
STDMETHODIMP CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)
@@ -71,6 +75,11 @@ STDMETHODIMP CLzmaEncoder::SetCoderProperties(const PROPID *propIDs, const PROPV
return S_OK;
}
STDMETHODIMP CLzmaEncoder::SetCoderPropertiesOpt(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps)
{
return EncoderSpec->SetCoderPropertiesOpt(propIDs, props, numProps);
}
STDMETHODIMP CLzmaEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)
{
@@ -118,7 +127,8 @@ HRESULT CAddCommon::CalcStreamCRC(ISequentialInStream *inStream, UInt32 &resultC
}
HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, CCompressingResult &opRes) const
HRESULT CAddCommon::Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize,
CCompressingResult &opRes) const
{
// We use Zip64, if unPackSize size is larger than 0xF8000000 to support
// cases when compressed size can be about 3% larger than uncompressed size
@@ -144,7 +154,7 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C
opRes.LzmaEos = false;
opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default;
opRes.FileTimeWasUsed = false;
opRes.DescriptorMode = outSeqMode;
if (_options.PasswordIsDefined)
{
@@ -153,8 +163,8 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C
opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Aes;
else
{
if (seqMode)
opRes.FileTimeWasUsed = true;
if (inSeqMode)
opRes.DescriptorMode = true;
}
}
@@ -187,7 +197,8 @@ HRESULT CAddCommon::Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, C
HRESULT CAddCommon::Compress(
DECL_EXTERNAL_CODECS_LOC_VARS
ISequentialInStream *inStream, IOutStream *outStream,
bool seqMode, UInt32 fileTime,
bool inSeqMode, bool outSeqMode,
UInt32 fileTime, UInt64 expectedDataSize,
ICompressProgressInfo *progress, CCompressingResult &opRes)
{
opRes.LzmaEos = false;
@@ -202,16 +213,31 @@ HRESULT CAddCommon::Compress(
CMyComPtr<ISequentialInStream> inCrcStream = inSecCrcStreamSpec;
CMyComPtr<IInStream> inStream2;
if (!seqMode)
if (!inSeqMode)
{
inStream->QueryInterface(IID_IInStream, (void **)&inStream2);
if (!inStream2)
{
// inSeqMode = true;
// inSeqMode must be correct before
return E_FAIL;
}
}
inSecCrcStreamSpec->SetStream(inStream);
inSecCrcStreamSpec->Init();
unsigned numTestMethods = _options.MethodSequence.Size();
if (seqMode || (numTestMethods > 1 && !inStream2))
numTestMethods = 1;
bool descriptorMode = outSeqMode;
if (!outSeqMode)
if (inSeqMode && _options.PasswordIsDefined && !_options.IsAesMode)
descriptorMode = true;
opRes.DescriptorMode = descriptorMode;
if (numTestMethods > 1)
if (inSeqMode || outSeqMode || !inStream2)
numTestMethods = 1;
UInt32 crc = 0;
bool crc_IsCalculated = false;
@@ -219,19 +245,23 @@ HRESULT CAddCommon::Compress(
Byte method = 0;
CFilterCoder::C_OutStream_Releaser outStreamReleaser;
opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default;
opRes.FileTimeWasUsed = false;
for (unsigned i = 0; i < numTestMethods; i++)
{
opRes.LzmaEos = false;
opRes.ExtractVersion = NCompressionMethod::kExtractVersion_Default;
if (inStream2 && i != 0)
if (i != 0)
{
inSecCrcStreamSpec->Init();
RINOK(inStream2->Seek(0, STREAM_SEEK_SET, NULL));
if (inStream2)
{
inSecCrcStreamSpec->Init();
RINOK(inStream2->Seek(0, STREAM_SEEK_SET, NULL));
}
RINOK(outStream->SetSize(0));
RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL));
}
RINOK(outStream->SetSize(0));
RINOK(outStream->Seek(0, STREAM_SEEK_SET, NULL));
if (_options.PasswordIsDefined)
{
@@ -264,7 +294,12 @@ HRESULT CAddCommon::Compress(
UInt32 check;
if (inStream2)
if (descriptorMode)
{
// it's Info-ZIP modification for stream_mode descriptor_mode (bit 3 of the general purpose bit flag is set)
check = (fileTime & 0xFFFF);
}
else
{
if (!crc_IsCalculated)
{
@@ -275,11 +310,6 @@ HRESULT CAddCommon::Compress(
}
check = (crc >> 16);
}
else
{
opRes.FileTimeWasUsed = true;
check = (fileTime & 0xFFFF);
}
RINOK(_filterSpec->WriteHeader_Check16(outStream, (UInt16)check));
}
@@ -295,7 +325,13 @@ HRESULT CAddCommon::Compress(
{
case NCompressionMethod::kStore:
{
if (_copyCoderSpec == NULL)
if (descriptorMode)
{
// we still can create descriptor_mode archives with "Store" method, but they are not good for 100%
return E_NOTIMPL;
}
if (!_copyCoderSpec)
{
_copyCoderSpec = new NCompress::CCopyCoder;
_copyCoder = _copyCoderSpec;
@@ -390,6 +426,18 @@ HRESULT CAddCommon::Compress(
outStreamNew = outStream;
if (_compressExtractVersion > opRes.ExtractVersion)
opRes.ExtractVersion = _compressExtractVersion;
{
CMyComPtr<ICompressSetCoderPropertiesOpt> optProps;
_compressEncoder->QueryInterface(IID_ICompressSetCoderPropertiesOpt, (void **)&optProps);
if (optProps)
{
PROPID propID = NCoderPropID::kExpectedDataSize;
NWindows::NCOM::CPropVariant prop = (UInt64)expectedDataSize;
RINOK(optProps->SetCoderPropertiesOpt(&propID, &prop, 1));
}
}
RINOK(_compressEncoder->Code(inCrcStream, outStreamNew, NULL, NULL, progress));
break;
}

8
CPP/7zip/Archive/Zip/ZipAddCommon.h
View File

@@ -26,7 +26,7 @@ struct CCompressingResult
UInt32 CRC;
UInt16 Method;
Byte ExtractVersion;
bool FileTimeWasUsed;
bool DescriptorMode;
bool LzmaEos;
};
@@ -53,12 +53,14 @@ public:
CAddCommon(const CCompressionMethodMode &options);
~CAddCommon();
HRESULT Set_Pre_CompressionResult(bool seqMode, UInt64 unpackSize, CCompressingResult &opRes) const;
HRESULT Set_Pre_CompressionResult(bool inSeqMode, bool outSeqMode, UInt64 unpackSize,
CCompressingResult &opRes) const;
HRESULT Compress(
DECL_EXTERNAL_CODECS_LOC_VARS
ISequentialInStream *inStream, IOutStream *outStream,
bool seqMode, UInt32 fileTime,
bool inSeqMode, bool outSeqMode,
UInt32 fileTime, UInt64 expectedDataSize,
ICompressProgressInfo *progress, CCompressingResult &opRes);
};

3
CPP/7zip/Archive/Zip/ZipHandler.cpp
View File

@@ -564,7 +564,8 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val
AString s2 = FlagsToString(g_HeaderCharacts, ARRAY_SIZE(g_HeaderCharacts), flags);
if (!s2.IsEmpty())
{
s.Add_OptSpaced(":");
if (!s.IsEmpty())
s.Add_OptSpaced(":");
s.Add_OptSpaced(s2);
}
}

28
CPP/7zip/Archive/Zip/ZipHandlerOut.cpp
View File

@@ -136,6 +136,7 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
thereAreAesUpdates = true;
if (!IntToBool(newProps))
ui.IsDir = inputItem.IsDir();
// ui.IsAltStream = inputItem.IsAltStream();
}
if (IntToBool(newProps))
@@ -175,6 +176,33 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt
ui.IsDir = (prop.boolVal != VARIANT_FALSE);
}
/*
{
bool isAltStream = false;
{
NCOM::CPropVariant prop;
RINOK(callback->GetProperty(i, kpidIsAltStream, &prop));
if (prop.vt == VT_BOOL)
isAltStream = (prop.boolVal != VARIANT_FALSE);
else if (prop.vt != VT_EMPTY)
return E_INVALIDARG;
}
if (isAltStream)
{
if (ui.IsDir)
return E_INVALIDARG;
int delim = name.ReverseFind(L':');
if (delim >= 0)
{
name.Delete(delim, 1);
name.Insert(delim, UString(k_SpecName_NTFS_STREAM));
ui.IsAltStream = true;
}
}
}
*/
{
CPropVariant prop;
RINOK(callback->GetProperty(i, kpidTimeType, &prop));

1
CPP/7zip/Archive/Zip/ZipHeader.h
View File

@@ -131,6 +131,7 @@ namespace NFileHeader
const unsigned kDescriptorUsedMask = 1 << 3;
const unsigned kStrongEncrypted = 1 << 6;
const unsigned kUtf8 = 1 << 11;
const unsigned kAltStream = 1 << 14;
const unsigned kImplodeDictionarySizeMask = 1 << 1;
const unsigned kImplodeLiteralsOnMask = 1 << 2;

11
CPP/7zip/Archive/Zip/ZipIn.cpp
View File

@@ -459,7 +459,16 @@ API_FUNC_IsArc IsArc_Zip(const Byte *p, size_t size)
extraSize -= 4;
p += 4;
if (dataSize > extraSize)
return k_IsArc_Res_NO;
{
// It can be error on header.
// We want to support such rare case bad archives.
// We use additional checks to reduce false-positive probability.
if (nameSize == 0
|| nameSize > (1 << 9)
|| extraSize > (1 << 9))
return k_IsArc_Res_NO;
return k_IsArc_Res_YES;
}
if (dataSize > size)
return k_IsArc_Res_NEED_MORE;
size -= dataSize;

5
CPP/7zip/Archive/Zip/ZipIn.h
View File

@@ -20,10 +20,13 @@ class CItemEx: public CItem
{
public:
UInt32 LocalFullHeaderSize; // including Name and Extra
// int ParentOfAltStream; // -1, if not AltStream
bool DescriptorWasRead;
CItemEx(): DescriptorWasRead(false) {}
CItemEx():
// ParentOfAltStream(-1),
DescriptorWasRead(false) {}
UInt64 GetLocalFullSize() const
{ return LocalFullHeaderSize + GetPackSizeWithDescriptor(); }

24
CPP/7zip/Archive/Zip/ZipItem.cpp
View File

@@ -20,6 +20,12 @@ namespace NZip {
using namespace NFileHeader;
/*
const char *k_SpecName_NTFS_STREAM = "@@NTFS@STREAM@";
const char *k_SpecName_MAC_RESOURCE_FORK = "@@MAC@RESOURCE-FORK@";
*/
static const CUInt32PCharPair g_ExtraTypes[] =
{
{ NExtraID::kZip64, "Zip64" },
@@ -269,7 +275,25 @@ UInt32 CItem::GetWinAttrib() const
case NHostOS::kUnix:
// do we need to clear 16 low bits in this case?
if (FromCentral)
{
/*
Some programs write posix attributes in high 16 bits of ExternalAttrib
Also some programs can write additional marker flag:
0x8000 - p7zip
0x4000 - Zip in MacOS
no marker - Info-Zip
Client code has two options to detect posix field:
1) check 0x8000 marker. In that case we must add 0x8000 marker here.
2) check that high 4 bits (file type bits in posix field) of attributes are not zero.
*/
winAttrib = ExternalAttrib & 0xFFFF0000;
// #ifndef _WIN32
winAttrib |= 0x8000; // add posix mode marker
// #endif
}
break;
}
if (IsDir()) // test it;

7
CPP/7zip/Archive/Zip/ZipItem.h
View File

@@ -14,6 +14,11 @@
namespace NArchive {
namespace NZip {
/*
extern const char *k_SpecName_NTFS_STREAM;
extern const char *k_SpecName_MAC_RESOURCE_FORK;
*/
struct CVersion
{
Byte Version;
@@ -233,6 +238,7 @@ public:
bool IsAesEncrypted() const { return IsEncrypted() && (IsStrongEncrypted() || Method == NFileHeader::NCompressionMethod::kWzAES); }
bool IsLzmaEOS() const { return (Flags & NFileHeader::NFlags::kLzmaEOS) != 0; }
bool HasDescriptor() const { return (Flags & NFileHeader::NFlags::kDescriptorUsedMask) != 0; }
// bool IsAltStream() const { return (Flags & NFileHeader::NFlags::kAltStream) != 0; }
unsigned GetDeflateLevel() const { return (Flags >> 1) & 3; }
@@ -264,6 +270,7 @@ public:
void ClearFlags() { Flags = 0; }
void SetEncrypted(bool encrypted) { SetFlag(NFileHeader::NFlags::kEncrypted, encrypted); }
void SetUtf8(bool isUtf8) { SetFlag(NFileHeader::NFlags::kUtf8, isUtf8); }
// void SetFlag_AltStream(bool isAltStream) { SetFlag(NFileHeader::NFlags::kAltStream, isAltStream); }
void SetDescriptorMode(bool useDescriptor) { SetFlag(NFileHeader::NFlags::kDescriptorUsedMask, useDescriptor); }
UINT GetCodePage() const { return CP_OEMCP; }

3
CPP/7zip/Archive/Zip/ZipOut.cpp
View File

@@ -174,6 +174,9 @@ void COutArchive::WriteLocalHeader_Replace(CItemOut &item)
{
WriteDescriptor(item);
m_OutBuffer.FlushWithCheck();
return;
// we don't replace local header, if we write Descriptor.
// so local header with Descriptor flag must be written to local header before.
}
const UInt64 nextPos = m_CurPos;

258
CPP/7zip/Archive/Zip/ZipUpdate.cpp
View File

@@ -62,7 +62,7 @@ static void AddAesExtra(CItem &item, Byte aesKeyMode, UInt16 method)
static void SetFileHeader(
const CCompressionMethodMode &options,
const CUpdateItem &ui,
// bool isSeqMode,
bool useDescriptor,
CItemOut &item)
{
item.Size = ui.Size;
@@ -75,6 +75,7 @@ static void SetFileHeader(
item.Name = ui.Name;
item.Comment = ui.Comment;
item.SetUtf8(ui.IsUtf8);
// item.SetFlag_AltStream(ui.IsAltStream);
item.ExternalAttrib = ui.Attrib;
item.Time = ui.Time;
item.Ntfs_MTime = ui.Ntfs_MTime;
@@ -94,7 +95,7 @@ static void SetFileHeader(
item.InternalAttrib = 0; // test it
item.SetEncrypted(!isDir && options.PasswordIsDefined);
// item.SetDescriptorMode(isSeqMode);
item.SetDescriptorMode(useDescriptor);
if (isDir)
{
@@ -166,18 +167,22 @@ struct CThreadInfo
HRESULT Result;
CCompressingResult CompressingResult;
bool SeqMode;
bool InSeqMode;
bool OutSeqMode;
bool IsFree;
UInt32 UpdateIndex;
UInt32 FileTime;
UInt64 ExpectedDataSize;
CThreadInfo(const CCompressionMethodMode &options):
ExitThread(false),
ProgressSpec(0),
OutStreamSpec(0),
Coder(options),
SeqMode(false),
FileTime(0)
InSeqMode(false),
OutSeqMode(false),
FileTime(0),
ExpectedDataSize((UInt64)(Int64)-1)
{}
HRESULT CreateEvents()
@@ -210,7 +215,9 @@ void CThreadInfo::WaitAndCode()
Result = Coder.Compress(
EXTERNAL_CODECS_LOC_VARS
InStream, OutStream, SeqMode, FileTime, Progress, CompressingResult);
InStream, OutStream,
InSeqMode, OutSeqMode, FileTime, ExpectedDataSize,
Progress, CompressingResult);
if (Result == S_OK && Progress)
Result = Progress->SetRatioInfo(&CompressingResult.UnpackSize, &CompressingResult.PackSize);
@@ -237,10 +244,13 @@ public:
struct CMemBlocks2: public CMemLockBlocks
{
CCompressingResult CompressingResult;
bool Defined;
bool Skip;
CMemBlocks2(): Defined(false), Skip(false) {}
bool InSeqMode;
bool PreDescriptorMode;
bool Finished;
CCompressingResult CompressingResult;
CMemBlocks2(): Skip(false), InSeqMode(false), PreDescriptorMode(false), Finished(false) {}
};
class CMemRefs
@@ -271,6 +281,7 @@ public:
MY_UNKNOWN_IMP
void Create(IProgress *progress, bool inSizeIsMain);
void SetProgressOffset(UInt64 progressOffset);
void SetProgressOffset_NoLock(UInt64 progressOffset);
HRESULT SetRatioInfo(unsigned index, const UInt64 *inSize, const UInt64 *outSize);
STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize);
};
@@ -283,11 +294,16 @@ void CMtProgressMixer2::Create(IProgress *progress, bool inSizeIsMain)
ProgressOffset = InSizes[0] = InSizes[1] = OutSizes[0] = OutSizes[1] = 0;
}
void CMtProgressMixer2::SetProgressOffset_NoLock(UInt64 progressOffset)
{
InSizes[1] = OutSizes[1] = 0;
ProgressOffset = progressOffset;
}
void CMtProgressMixer2::SetProgressOffset(UInt64 progressOffset)
{
CriticalSection.Enter();
InSizes[1] = OutSizes[1] = 0;
ProgressOffset = progressOffset;
SetProgressOffset_NoLock(progressOffset);
CriticalSection.Leave();
}
@@ -375,6 +391,7 @@ static HRESULT UpdateItemOldData(
item.Comment = ui.Comment;
item.Name = ui.Name;
item.SetUtf8(ui.IsUtf8);
// item.SetFlag_AltStream(ui.IsAltStream);
item.Time = ui.Time;
item.Ntfs_MTime = ui.Ntfs_MTime;
item.Ntfs_ATime = ui.Ntfs_ATime;
@@ -412,7 +429,7 @@ static HRESULT UpdateItemOldData(
static void WriteDirHeader(COutArchive &archive, const CCompressionMethodMode *options,
const CUpdateItem &ui, CItemOut &item)
{
SetFileHeader(*options, ui, item);
SetFileHeader(*options, ui, false, item);
archive.WriteLocalHeader(item);
}
@@ -471,7 +488,7 @@ static HRESULT Update2St(
CInArchive *inArchive,
const CObjectVector<CItemEx> &inputItems,
CObjectVector<CUpdateItem> &updateItems,
const CCompressionMethodMode *options,
const CCompressionMethodMode *options, bool outSeqMode,
const CByteBuffer *comment,
IArchiveUpdateCallback *updateCallback,
UInt64 &totalComplexity,
@@ -527,28 +544,28 @@ static HRESULT Update2St(
if (!fileInStream)
return E_INVALIDARG;
bool seqMode;
bool inSeqMode = false;
if (!inSeqMode)
{
CMyComPtr<IInStream> inStream2;
fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
seqMode = (inStream2 == NULL);
inSeqMode = (inStream2 == NULL);
}
// seqMode = true; // to test seqMode
UpdatePropsFromStream(ui, fileInStream, updateCallback, totalComplexity);
SetFileHeader(*options, ui, item);
item.SetDescriptorMode(seqMode);
// file Size can be 64-bit !!!
CCompressingResult compressingResult;
RINOK(compressor.Set_Pre_CompressionResult(
seqMode,
inSeqMode, outSeqMode,
ui.Size,
compressingResult));
SetFileHeader(*options, ui, compressingResult.DescriptorMode, item);
// file Size can be 64-bit !!!
SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
archive.WriteLocalHeader(item);
@@ -559,17 +576,12 @@ static HRESULT Update2St(
RINOK(compressor.Compress(
EXTERNAL_CODECS_LOC_VARS
fileInStream, outStream,
seqMode, ui.Time,
inSeqMode, outSeqMode,
ui.Time, ui.Size,
progress, compressingResult));
if (compressingResult.FileTimeWasUsed)
{
/*
if (!item.HasDescriptor())
return E_FAIL;
*/
item.SetDescriptorMode(true);
}
if (item.HasDescriptor() != compressingResult.DescriptorMode)
return E_FAIL;
SetItemInfoFromCompressingResult(compressingResult, options->IsRealAesMode(), options->AesKeyMode, item);
@@ -598,8 +610,11 @@ static HRESULT Update2St(
lps->InSize = unpackSizeTotal;
lps->OutSize = packSizeTotal;
RINOK(lps->SetCur());
archive.WriteCentralDir(items, comment);
return S_OK;
lps->ProgressOffset += kCentralHeaderSize * updateItems.Size() + 1;
return lps->SetCur();
}
@@ -609,7 +624,7 @@ static HRESULT Update2(
CInArchive *inArchive,
const CObjectVector<CItemEx> &inputItems,
CObjectVector<CUpdateItem> &updateItems,
const CCompressionMethodMode &options,
const CCompressionMethodMode &options, bool outSeqMode,
const CByteBuffer *comment,
IArchiveUpdateCallback *updateCallback)
{
@@ -743,32 +758,38 @@ static HRESULT Update2(
const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
const UInt32 blockSize = oneMethodMain->Get_BZip2_BlockSize();
const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1;
numBZip2Threads = 32;
numBZip2Threads = 64;
if (numBZip2Threads > averageNumberOfBlocks)
numBZip2Threads = (UInt32)averageNumberOfBlocks;
if (numBZip2Threads > numThreads)
numBZip2Threads = numThreads;
oneMethodMain->AddProp_NumThreads(numBZip2Threads);
}
numThreads /= numBZip2Threads;
}
if (method == NFileHeader::NCompressionMethod::kXz)
else if (method == NFileHeader::NCompressionMethod::kXz)
{
bool fixedNumber;
UInt32 numLzma2Threads = oneMethodMain->Get_Lzma2_NumThreads(fixedNumber);
if (!fixedNumber)
UInt32 numLzmaThreads = 1;
int numXzThreads = oneMethodMain->Get_Xz_NumThreads(numLzmaThreads);
if (numXzThreads < 0)
{
const UInt64 averageSize = numBytesToCompress / numFilesToCompress;
const UInt64 blockSize = oneMethodMain->Get_Lzma2_BlockSize();
const UInt64 averageNumberOfBlocks = averageSize / blockSize + 1;
numLzma2Threads = 2;
if (numLzma2Threads > averageNumberOfBlocks)
numLzma2Threads = (UInt32)averageNumberOfBlocks;
oneMethodMain->AddProp_NumThreads(numLzma2Threads);
const UInt64 blockSize = oneMethodMain->Get_Xz_BlockSize();
UInt64 averageNumberOfBlocks = 1;
if (blockSize != (UInt64)(Int64)-1)
averageNumberOfBlocks = averageSize / blockSize + 1;
UInt32 t = 256;
if (t > averageNumberOfBlocks)
t = (UInt32)averageNumberOfBlocks;
t *= numLzmaThreads;
if (t > numThreads)
t = numThreads;
oneMethodMain->AddProp_NumThreads(t);
numXzThreads = t;
}
numThreads /= numLzma2Threads;
numThreads /= (unsigned)numXzThreads;
}
if (method == NFileHeader::NCompressionMethod::kLZMA)
else if (method == NFileHeader::NCompressionMethod::kLZMA)
{
// we suppose that default LZMA is 2 thread. So we don't change it
UInt32 numLZMAThreads = oneMethodMain->Get_Lzma_NumThreads();
@@ -782,12 +803,14 @@ static HRESULT Update2(
mtMode = false;
}
// mtMode = true; // to test mtMode for seqMode
if (!mtMode)
#endif
return Update2St(
EXTERNAL_CODECS_LOC_VARS
archive, inArchive,
inputItems, updateItems, &options2, comment, updateCallback, totalComplexity, opCallback);
inputItems, updateItems, &options2, outSeqMode, comment, updateCallback, totalComplexity, opCallback);
#ifndef _7ZIP_ST
@@ -830,8 +853,10 @@ static HRESULT Update2(
threadInfo.ProgressSpec = new CMtCompressProgress();
threadInfo.Progress = threadInfo.ProgressSpec;
threadInfo.ProgressSpec->Init(&mtCompressProgressMixer, (int)i);
threadInfo.SeqMode = false; // fix it !
threadInfo.FileTime = 0; // fix it !
threadInfo.InSeqMode = false;
threadInfo.OutSeqMode = false;
threadInfo.FileTime = 0;
threadInfo.ExpectedDataSize = (UInt64)(Int64)-1;
RINOK(threadInfo.CreateThread());
}
}
@@ -840,10 +865,15 @@ static HRESULT Update2(
unsigned itemIndex = 0;
int lastRealStreamItemIndex = -1;
while (itemIndex < updateItems.Size())
{
if (threadIndices.Size() < numThreads && mtItemIndex < updateItems.Size())
{
// we start ahead the threads for compressing
// also we set refs.Refs[itemIndex].SeqMode that is used later
// don't move that code block
CUpdateItem &ui = updateItems[mtItemIndex++];
if (!ui.NewData)
continue;
@@ -869,6 +899,8 @@ static HRESULT Update2(
CMyComPtr<ISequentialInStream> fileInStream;
CMemBlocks2 &memRef2 = refs.Refs[mtItemIndex - 1];
{
NWindows::NSynchronization::CCriticalSectionLock lock(mtProgressMixerSpec->Mixer2->CriticalSection);
HRESULT res = updateCallback->GetStream(ui.IndexInClient, &fileInStream);
@@ -876,9 +908,9 @@ static HRESULT Update2(
{
complexity += ui.Size;
complexity += kLocalHeaderSize;
mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
mtProgressMixerSpec->Mixer2->SetProgressOffset_NoLock(complexity);
RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK));
refs.Refs[mtItemIndex - 1].Skip = true;
memRef2.Skip = true;
continue;
}
RINOK(res);
@@ -888,26 +920,46 @@ static HRESULT Update2(
RINOK(updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK));
}
for (UInt32 k = 0; k < numThreads; k++)
UInt32 k;
for (k = 0; k < numThreads; k++)
if (threads.Threads[k].IsFree)
break;
if (k == numThreads)
return E_FAIL;
{
CThreadInfo &threadInfo = threads.Threads[k];
if (threadInfo.IsFree)
{
CThreadInfo &threadInfo = threads.Threads[k];
threadInfo.IsFree = false;
threadInfo.InStream = fileInStream;
bool inSeqMode = false;
if (!inSeqMode)
{
CMyComPtr<IInStream> inStream2;
fileInStream->QueryInterface(IID_IInStream, (void **)&inStream2);
inSeqMode = (inStream2 == NULL);
}
memRef2.InSeqMode = inSeqMode;
// !!!!! we must release ref before sending event
// BUG was here in v4.43 and v4.44. It could change ref counter in two threads in same time
fileInStream.Release();
threadInfo.OutStreamSpec->Init();
threadInfo.ProgressSpec->Reinit();
threadInfo.CompressEvent.Set();
threadInfo.UpdateIndex = mtItemIndex - 1;
threadInfo.InSeqMode = inSeqMode;
threadInfo.OutSeqMode = outSeqMode;
threadInfo.FileTime = ui.Time; // FileTime is used for ZipCrypto only in seqMode
threadInfo.ExpectedDataSize = ui.Size;
threadInfo.CompressEvent.Set();
compressingCompletedEvents.Add(threadInfo.CompressionCompletedEvent);
threadIndices.Add(k);
break;
}
}
@@ -946,15 +998,17 @@ static HRESULT Update2(
{
CMemBlocks2 &memRef = refs.Refs[itemIndex];
if (memRef.Defined)
if (memRef.Finished)
{
if (lastRealStreamItemIndex < (int)itemIndex)
lastRealStreamItemIndex = itemIndex;
SetFileHeader(options, ui, item);
SetFileHeader(options, ui, memRef.CompressingResult.DescriptorMode, item);
// the BUG was fixed in 9.26:
// SetItemInfoFromCompressingResult must be after SetFileHeader
// to write correct Size.
SetItemInfoFromCompressingResult(memRef.CompressingResult,
options.IsRealAesMode(), options.AesKeyMode, item);
archive.WriteLocalHeader(item);
@@ -967,16 +1021,25 @@ static HRESULT Update2(
}
else
{
// current file was not finished
if (lastRealStreamItemIndex < (int)itemIndex)
{
// LocalHeader was not written for current itemIndex still
lastRealStreamItemIndex = itemIndex;
SetFileHeader(options, ui, item);
// thread was started before for that item already, and memRef.SeqMode was set
CCompressingResult compressingResult;
RINOK(compressor.Set_Pre_CompressionResult(
false, // seqMode
memRef.InSeqMode, outSeqMode,
ui.Size,
compressingResult));
memRef.PreDescriptorMode = compressingResult.DescriptorMode;
SetFileHeader(options, ui, compressingResult.DescriptorMode, item);
SetItemInfoFromCompressingResult(compressingResult, options.IsRealAesMode(), options.AesKeyMode, item);
// file Size can be 64-bit !!!
@@ -1015,19 +1078,29 @@ static HRESULT Update2(
if (t == 0)
{
// if thread for current file was finished.
if (threadInfo.UpdateIndex != itemIndex)
return E_FAIL;
if (memRef.PreDescriptorMode != threadInfo.CompressingResult.DescriptorMode)
return E_FAIL;
RINOK(threadInfo.OutStreamSpec->WriteToRealStream());
threadInfo.OutStreamSpec->ReleaseOutStream();
SetFileHeader(options, ui, item);
SetFileHeader(options, ui, threadInfo.CompressingResult.DescriptorMode, item);
SetItemInfoFromCompressingResult(threadInfo.CompressingResult,
options.IsRealAesMode(), options.AesKeyMode, item);
archive.WriteLocalHeader_Replace(item);
}
else
{
// it's not current file. So we must store information in array
CMemBlocks2 &memRef2 = refs.Refs[threadInfo.UpdateIndex];
threadInfo.OutStreamSpec->DetachData(memRef2);
memRef2.CompressingResult = threadInfo.CompressingResult;
memRef2.Defined = true;
// memRef2.SeqMode = threadInfo.SeqMode; // it was set before
memRef2.Finished = true;
continue;
}
}
@@ -1045,8 +1118,13 @@ static HRESULT Update2(
}
RINOK(mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL));
archive.WriteCentralDir(items, comment);
return S_OK;
complexity += kCentralHeaderSize * updateItems.Size() + 1;
mtProgressMixerSpec->Mixer2->SetProgressOffset(complexity);
return mtCompressProgressMixer.SetRatioInfo(0, NULL, NULL);
#endif
}
@@ -1060,6 +1138,7 @@ class CCacheOutStream:
public CMyUnknownImp
{
CMyComPtr<IOutStream> _stream;
CMyComPtr<ISequentialOutStream> _seqStream;
Byte *_cache;
UInt64 _virtPos;
UInt64 _virtSize;
@@ -1075,10 +1154,10 @@ class CCacheOutStream:
}
HRESULT FlushCache();
public:
CCacheOutStream(): _cache(0) {}
CCacheOutStream(): _cache(NULL) {}
~CCacheOutStream();
bool Allocate();
HRESULT Init(IOutStream *stream);
HRESULT Init(ISequentialOutStream *seqStream, IOutStream *stream);
MY_UNKNOWN_IMP
@@ -1094,13 +1173,19 @@ bool CCacheOutStream::Allocate()
return (_cache != NULL);
}
HRESULT CCacheOutStream::Init(IOutStream *stream)
HRESULT CCacheOutStream::Init(ISequentialOutStream *seqStream, IOutStream *stream)
{
_virtPos = _phyPos = 0;
_virtPos = 0;
_phyPos = 0;
_virtSize = 0;
_seqStream = seqStream;
_stream = stream;
RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos));
RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize));
RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos));
if (_stream)
{
RINOK(_stream->Seek(0, STREAM_SEEK_CUR, &_virtPos));
RINOK(_stream->Seek(0, STREAM_SEEK_END, &_virtSize));
RINOK(_stream->Seek(_virtPos, STREAM_SEEK_SET, &_virtPos));
}
_phyPos = _virtPos;
_phySize = _virtSize;
_cachedPos = 0;
@@ -1114,12 +1199,14 @@ HRESULT CCacheOutStream::MyWrite(size_t size)
{
if (_phyPos != _cachedPos)
{
if (!_stream)
return E_FAIL;
RINOK(_stream->Seek(_cachedPos, STREAM_SEEK_SET, &_phyPos));
}
size_t pos = (size_t)_cachedPos & kCacheMask;
size_t curSize = MyMin(kCacheSize - pos, _cachedSize);
curSize = MyMin(curSize, size);
RINOK(WriteStream(_stream, _cache + pos, curSize));
RINOK(WriteStream(_seqStream, _cache + pos, curSize));
_phyPos += curSize;
if (_phySize < _phyPos)
_phySize = _phyPos;
@@ -1138,10 +1225,13 @@ HRESULT CCacheOutStream::FlushCache()
CCacheOutStream::~CCacheOutStream()
{
FlushCache();
if (_virtSize != _phySize)
_stream->SetSize(_virtSize);
if (_virtPos != _phyPos)
_stream->Seek(_virtPos, STREAM_SEEK_SET, NULL);
if (_stream)
{
if (_virtSize != _phySize)
_stream->SetSize(_virtSize);
if (_virtPos != _phyPos)
_stream->Seek(_virtPos, STREAM_SEEK_SET, NULL);
}
::MidFree(_cache);
}
@@ -1250,6 +1340,8 @@ STDMETHODIMP CCacheOutStream::SetSize(UInt64 newSize)
_virtSize = newSize;
if (newSize < _phySize)
{
if (!_stream)
return E_NOTIMPL;
RINOK(_stream->SetSize(newSize));
_phySize = newSize;
}
@@ -1281,11 +1373,14 @@ HRESULT Update(
CMyComPtr<IOutStream> outStream;
bool outSeqMode;
{
CMyComPtr<IOutStream> outStreamReal;
seqOutStream->QueryInterface(IID_IOutStream, (void **)&outStreamReal);
if (!outStreamReal)
return E_NOTIMPL;
{
// return E_NOTIMPL;
}
if (inArchive)
{
@@ -1293,7 +1388,7 @@ HRESULT Update(
{
IInStream *baseStream = inArchive->GetBaseStream();
RINOK(baseStream->Seek(0, STREAM_SEEK_SET, NULL));
RINOK(NCompress::CopyStream_ExactSize(baseStream, outStreamReal, inArchive->ArcInfo.Base, NULL));
RINOK(NCompress::CopyStream_ExactSize(baseStream, seqOutStream, inArchive->ArcInfo.Base, NULL));
}
}
@@ -1301,7 +1396,8 @@ HRESULT Update(
outStream = cacheStream;
if (!cacheStream->Allocate())
return E_OUTOFMEMORY;
RINOK(cacheStream->Init(outStreamReal));
RINOK(cacheStream->Init(seqOutStream, outStreamReal));
outSeqMode = (outStreamReal == NULL);
}
COutArchive outArchive;
@@ -1323,7 +1419,7 @@ HRESULT Update(
EXTERNAL_CODECS_LOC_VARS
outArchive, inArchive,
inputItems, updateItems,
compressionMethodMode,
compressionMethodMode, outSeqMode,
inArchive ? &inArchive->ArcInfo.Comment : NULL,
updateCallback);
}

10
CPP/7zip/Archive/Zip/ZipUpdate.h
View File

@@ -32,6 +32,7 @@ struct CUpdateItem
bool IsDir;
bool NtfsTimeIsDefined;
bool IsUtf8;
// bool IsAltStream;
int IndexInArc;
int IndexInClient;
UInt32 Attrib;
@@ -50,12 +51,19 @@ struct CUpdateItem
IsDir = false;
NtfsTimeIsDefined = false;
IsUtf8 = false;
// IsAltStream = false;
Size = 0;
Name.Empty();
Comment.Free();
}
CUpdateItem(): NtfsTimeIsDefined(false), IsUtf8(false), Size(0) {}
CUpdateItem():
IsDir(false),
NtfsTimeIsDefined(false),
IsUtf8(false),
// IsAltStream(false),
Size(0)
{}
};
HRESULT Update(

1
CPP/7zip/Bundles/Format7zF/Arc.mak
View File

@@ -198,6 +198,7 @@ COMPRESS_OBJS = \
$O\DeflateRegister.obj \
$O\DeltaFilter.obj \
$O\ImplodeDecoder.obj \
$O\LzfseDecoder.obj \
$O\LzhDecoder.obj \
$O\Lzma2Decoder.obj \
$O\Lzma2Encoder.obj \

18
CPP/7zip/Bundles/Format7zF/Format7z.dsp
View File

@@ -961,6 +961,24 @@ SOURCE=..\..\Compress\HuffmanDecoder.h
# End Source File
# Begin Source File
SOURCE=..\..\Compress\LzfseDecoder.cpp
!IF "$(CFG)" == "7z - Win32 Release"
# ADD CPP /O2
# SUBTRACT CPP /YX /Yc /Yu
!ELSEIF "$(CFG)" == "7z - Win32 Debug"
!ENDIF
# End Source File
# Begin Source File
SOURCE=..\..\Compress\LzfseDecoder.h
# End Source File
# Begin Source File
SOURCE=..\..\Compress\LzhDecoder.cpp
# End Source File
# Begin Source File

17
CPP/7zip/Common/CWrappers.cpp
View File

@@ -28,13 +28,26 @@ HRESULT SResToHRESULT(SRes res) throw()
switch (res)
{
case SZ_OK: return S_OK;
case SZ_ERROR_DATA:
case SZ_ERROR_CRC:
case SZ_ERROR_INPUT_EOF:
return S_FALSE;
case SZ_ERROR_MEM: return E_OUTOFMEMORY;
case SZ_ERROR_PARAM: return E_INVALIDARG;
case SZ_ERROR_PROGRESS: return E_ABORT;
case SZ_ERROR_DATA: return S_FALSE;
case SZ_ERROR_UNSUPPORTED: return E_NOTIMPL;
// case SZ_ERROR_READ: return E_NOTIMPL;
// case SZ_ERROR_OUTPUT_EOF:
// case SZ_ERROR_READ:
// case SZ_ERROR_WRITE:
// case SZ_ERROR_THREAD:
// case SZ_ERROR_ARCHIVE:
// case SZ_ERROR_NO_ARCHIVE:
// return E_FAIL;
}
if (res < 0)
return res;
return E_FAIL;
}

28
CPP/7zip/Common/InBuffer.cpp
View File

@@ -97,6 +97,33 @@ Byte CInBufferBase::ReadByte_FromNewBlock()
size_t CInBufferBase::ReadBytes(Byte *buf, size_t size)
{
size_t num = 0;
for (;;)
{
const size_t rem = _bufLim - _buf;
if (size <= rem)
{
if (size != 0)
{
memcpy(buf, _buf, size);
_buf += size;
num += size;
}
return num;
}
if (rem != 0)
{
memcpy(buf, _buf, rem);
_buf += rem;
buf += rem;
num += rem;
size -= rem;
}
if (!ReadBlock())
return num;
}
/*
if ((size_t)(_bufLim - _buf) >= size)
{
const Byte *src = _buf;
@@ -113,6 +140,7 @@ size_t CInBufferBase::ReadBytes(Byte *buf, size_t size)
buf[i] = *_buf++;
}
return size;
*/
}
size_t CInBufferBase::Skip(size_t size)

36
CPP/7zip/Common/MethodProps.cpp
View File

@@ -42,6 +42,14 @@ unsigned ParseStringToUInt32(const UString &srcString, UInt32 &number)
return (unsigned)(end - start);
}
static unsigned ParseStringToUInt64(const UString &srcString, UInt64 &number)
{
const wchar_t *start = srcString;
const wchar_t *end;
number = ConvertStringToUInt64(start, &end);
return (unsigned)(end - start);
}
HRESULT ParsePropToUInt32(const UString &name, const PROPVARIANT &prop, UInt32 &resValue)
{
// =VT_UI4
@@ -263,7 +271,11 @@ static const CNameToPropID g_NameToPropID[] =
{ VT_UI4, "mt" },
{ VT_BOOL, "eos" },
{ VT_UI4, "x" },
{ VT_UI4, "reduceSize" }
{ VT_UI8, "reduce" },
{ VT_UI8, "expect" },
{ VT_UI4, "b" },
{ VT_UI4, "check" },
{ VT_BSTR, "filter" }
};
static int FindPropIdExact(const UString &name)
@@ -345,7 +357,8 @@ static bool IsLogSizeProp(PROPID propid)
case NCoderPropID::kDictionarySize:
case NCoderPropID::kUsedMemorySize:
case NCoderPropID::kBlockSize:
case NCoderPropID::kReduceSize:
case NCoderPropID::kBlockSize2:
// case NCoderPropID::kReduceSize:
return true;
}
return false;
@@ -378,9 +391,22 @@ HRESULT CMethodProps::SetParam(const UString &name, const UString &value)
}
else if (!value.IsEmpty())
{
UInt32 number;
if (ParseStringToUInt32(value, number) == value.Len())
propValue = number;
if (nameToPropID.VarType == VT_UI4)
{
UInt32 number;
if (ParseStringToUInt32(value, number) == value.Len())
propValue = number;
else
propValue = value;
}
else if (nameToPropID.VarType == VT_UI8)
{
UInt64 number;
if (ParseStringToUInt64(value, number) == value.Len())
propValue = number;
else
propValue = value;
}
else
propValue = value;
}

47
CPP/7zip/Common/MethodProps.h
View File

@@ -4,6 +4,7 @@
#define __7Z_METHOD_PROPS_H
#include "../../Common/MyString.h"
#include "../../Common/Defs.h"
#include "../../Windows/Defs.h"
@@ -133,38 +134,50 @@ public:
return 2;
}
UInt32 Get_Lzma2_NumThreads(bool &fixedNumber) const
int Get_Xz_NumThreads(UInt32 &lzmaThreads) const
{
fixedNumber = false;
lzmaThreads = 1;
int numThreads = Get_NumThreads();
if (numThreads >= 0)
{
fixedNumber = true;
if (numThreads < 1) return 1;
const unsigned kNumLzma2ThreadsMax = 32;
if (numThreads > kNumLzma2ThreadsMax) return kNumLzma2ThreadsMax;
return numThreads;
}
return 1;
if (numThreads >= 0 && numThreads <= 1)
return 1;
if (Get_Lzma_Algo() != 0)
lzmaThreads = 2;
return numThreads;
}
UInt64 Get_Lzma2_BlockSize() const
UInt64 GetProp_BlockSize(PROPID id) const
{
int i = FindProp(NCoderPropID::kBlockSize);
int i = FindProp(id);
if (i >= 0)
{
const NWindows::NCOM::CPropVariant &val = Props[i].Value;
if (val.vt == VT_UI4) return val.ulVal;
if (val.vt == VT_UI8) return val.uhVal.QuadPart;
if (val.vt == VT_UI4) { return val.ulVal; }
if (val.vt == VT_UI8) { return val.uhVal.QuadPart; }
}
return 0;
}
UInt32 dictSize = Get_Lzma_DicSize();
UInt64 blockSize = (UInt64)dictSize << 2;
UInt64 Get_Xz_BlockSize() const
{
{
UInt64 blockSize1 = GetProp_BlockSize(NCoderPropID::kBlockSize);
UInt64 blockSize2 = GetProp_BlockSize(NCoderPropID::kBlockSize2);
UInt64 minSize = MyMin(blockSize1, blockSize2);
if (minSize != 0)
return minSize;
UInt64 maxSize = MyMax(blockSize1, blockSize2);
if (maxSize != 0)
return maxSize;
}
const UInt32 kMinSize = (UInt32)1 << 20;
const UInt32 kMaxSize = (UInt32)1 << 28;
UInt32 dictSize = Get_Lzma_DicSize();
UInt64 blockSize = (UInt64)dictSize << 2;
if (blockSize < kMinSize) blockSize = kMinSize;
if (blockSize > kMaxSize) blockSize = kMaxSize;
if (blockSize < dictSize) blockSize = dictSize;
blockSize += (kMinSize - 1);
blockSize &= ~(UInt64)(kMinSize - 1);
return blockSize;
}

11
CPP/7zip/Compress/BZip2Const.h
View File

@@ -50,6 +50,17 @@ const UInt32 kNumSelectorsMax = (2 + (kBlockSizeMax / kGroupSize));
const unsigned kRleModeRepSize = 4;
/*
The number of selectors stored in bzip2 block:
(numSelectors <= 18001) - must work with any decoder.
(numSelectors == 18002) - works with bzip2 1.0.6 decoder and all derived decoders.
(numSelectors > 18002)
7-Zip decoder doesn't support it.
bzip2 1.0.6 decoder can overflow selector[18002] arrays. But there are another
arrays after selector arrays. So the compiled code works.
lbzip2 2.5 encoder can write up to (18001 + 7) selectors.
*/
}}
#endif

6
CPP/7zip/Compress/BZip2Decoder.cpp
View File

@@ -411,9 +411,13 @@ SRes CBase::ReadBlock2()
lens[state4] = (Byte)state3;
state5 = 0;
}
// lbzip2 2.5 can produce dummy tree, where lens[i] = kMaxHuffmanLen
// BuildFull() returns error for such tree
/*
for (unsigned i = state4; i < kMaxAlphaSize; i++)
lens[i] = 0;
if (!huffs[state2].Build(lens))
*/
if (!huffs[state2].BuildFull(lens, state4))
return SZ_ERROR_DATA;
@@ -474,7 +478,7 @@ SRes CBase::ReadBlock2()
for (len = kNumTableBits + 1; val >= huff->_limits[len]; len++);
/*
if (len > kNumBitsMax)
return SZ_ERROR_DATA;
return SZ_ERROR_DATA; // that check is required, if NHuffman::Build() was used instead of BuildFull()
*/
if (_numBits < len)
{

33
CPP/7zip/Compress/LzOutWindow.h
View File

@@ -56,6 +56,39 @@ public:
if (pos == _limitPos)
FlushWithCheck();
}
void PutBytes(const Byte *data, UInt32 size)
{
if (size == 0)
return;
UInt32 pos = _pos;
Byte *buf = _buf;
buf[pos++] = *data++;
size--;
for (;;)
{
UInt32 limitPos = _limitPos;
UInt32 rem = limitPos - pos;
if (rem == 0)
{
_pos = pos;
FlushWithCheck();
pos = _pos;
continue;
}
if (size == 0)
break;
if (rem > size)
rem = size;
size -= rem;
do
buf[pos++] = *data++;
while (--rem);
}
_pos = pos;
}
Byte GetByte(UInt32 distance) const
{

925
CPP/7zip/Compress/LzfseDecoder.cpp Normal file
View File

@@ -0,0 +1,925 @@
// LzfseDecoder.cpp
/*
This code implements LZFSE data decompressing.
The code from "LZFSE compression library" was used.
2018 : Igor Pavlov : BSD 3-clause License : the code in this file
2015-2017 : Apple Inc : BSD 3-clause License : original "LZFSE compression library" code
The code in the "LZFSE compression library" is licensed under the "BSD 3-clause License":
----
Copyright (c) 2015-2016, Apple Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
----
*/
#include "StdAfx.h"
// #define SHOW_DEBUG_INFO
#ifdef SHOW_DEBUG_INFO
#include <stdio.h>
#endif
#ifdef SHOW_DEBUG_INFO
#define PRF(x) x
#else
#define PRF(x)
#endif
#include "../../../C/CpuArch.h"
#include "LzfseDecoder.h"
namespace NCompress {
namespace NLzfse {
static const Byte kSignature_LZFSE_V1 = 0x31; // '1'
static const Byte kSignature_LZFSE_V2 = 0x32; // '2'
HRESULT CDecoder::GetUInt32(UInt32 &val)
{
Byte b[4];
for (unsigned i = 0; i < 4; i++)
if (!m_InStream.ReadByte(b[i]))
return S_FALSE;
val = GetUi32(b);
return S_OK;
}
HRESULT CDecoder::DecodeUncompressed(UInt32 unpackSize)
{
PRF(printf("\nUncompressed %7u\n", unpackSize));
const unsigned kBufSize = 1 << 8;
Byte buf[kBufSize];
for (;;)
{
if (unpackSize == 0)
return S_OK;
UInt32 cur = unpackSize;
if (cur > kBufSize)
cur = kBufSize;
UInt32 cur2 = (UInt32)m_InStream.ReadBytes(buf, cur);
m_OutWindowStream.PutBytes(buf, cur2);
if (cur != cur2)
return S_FALSE;
}
}
HRESULT CDecoder::DecodeLzvn(UInt32 unpackSize)
{
UInt32 packSize;
RINOK(GetUInt32(packSize));
PRF(printf("\nLZVN %7u %7u", unpackSize, packSize));
UInt32 D = 0;
for (;;)
{
if (packSize == 0)
return S_FALSE;
Byte b;
if (!m_InStream.ReadByte(b))
return S_FALSE;
packSize--;
UInt32 M;
UInt32 L;
if (b >= 0xE0)
{
/*
large L - 11100000 LLLLLLLL <LITERALS>
small L - 1110LLLL <LITERALS>
large Rep - 11110000 MMMMMMMM
small Rep - 1111MMMM
*/
M = b & 0xF;
if (M == 0)
{
if (packSize == 0)
return S_FALSE;
Byte b1;
if (!m_InStream.ReadByte(b1))
return S_FALSE;
packSize--;
M = (UInt32)b1 + 16;
}
L = 0;
if ((b & 0x10) == 0)
{
// Literals only
L = M;
M = 0;
}
}
// ERROR codes
else if ((b & 0xF0) == 0x70) // 0111xxxx
return S_FALSE;
else if ((b & 0xF0) == 0xD0) // 1101xxxx
return S_FALSE;
else
{
if ((b & 0xE0) == 0xA0)
{
// medium - 101LLMMM DDDDDDMM DDDDDDDD <LITERALS>
if (packSize < 2)
return S_FALSE;
Byte b1;
if (!m_InStream.ReadByte(b1))
return S_FALSE;
packSize--;
Byte b2;
if (!m_InStream.ReadByte(b2))
return S_FALSE;
packSize--;
L = (((UInt32)b >> 3) & 3);
M = (((UInt32)b & 7) << 2) + (b1 & 3);
D = ((UInt32)b1 >> 2) + ((UInt32)b2 << 6);
}
else
{
L = (UInt32)b >> 6;
M = ((UInt32)b >> 3) & 7;
if ((b & 0x7) == 6)
{
// REP - LLMMM110 <LITERALS>
if (L == 0)
{
// spec
if (M == 0)
break; // EOS
if (M <= 2)
continue; // NOP
return S_FALSE; // UNDEFINED
}
}
else
{
if (packSize == 0)
return S_FALSE;
Byte b1;
if (!m_InStream.ReadByte(b1))
return S_FALSE;
packSize--;
// large - LLMMM111 DDDDDDDD DDDDDDDD <LITERALS>
// small - LLMMMDDD DDDDDDDD <LITERALS>
D = ((UInt32)b & 7);
if (D == 7)
{
if (packSize == 0)
return S_FALSE;
Byte b2;
if (!m_InStream.ReadByte(b2))
return S_FALSE;
packSize--;
D = b2;
}
D = (D << 8) + b1;
}
}
M += 3;
}
{
for (unsigned i = 0; i < L; i++)
{
if (packSize == 0 || unpackSize == 0)
return S_FALSE;
Byte b1;
if (!m_InStream.ReadByte(b1))
return S_FALSE;
packSize--;
m_OutWindowStream.PutByte(b1);
unpackSize--;
}
}
if (M != 0)
{
if (unpackSize == 0 || D == 0)
return S_FALSE;
unsigned cur = M;
if (cur > unpackSize)
cur = (unsigned)unpackSize;
if (!m_OutWindowStream.CopyBlock(D - 1, cur))
return S_FALSE;
unpackSize -= cur;
if (cur != M)
return S_FALSE;
}
}
if (unpackSize != 0)
return S_FALSE;
// LZVN encoder writes 7 additional zero bytes
if (packSize != 7)
return S_FALSE;
do
{
Byte b;
if (!m_InStream.ReadByte(b))
return S_FALSE;
packSize--;
if (b != 0)
return S_FALSE;
}
while (packSize != 0);
return S_OK;
}
// ---------- LZFSE ----------
#define MATCHES_PER_BLOCK 10000
#define LITERALS_PER_BLOCK (4 * MATCHES_PER_BLOCK)
#define NUM_L_SYMBOLS 20
#define NUM_M_SYMBOLS 20
#define NUM_D_SYMBOLS 64
#define NUM_LIT_SYMBOLS 256
#define NUM_SYMBOLS ( \
NUM_L_SYMBOLS + \
NUM_M_SYMBOLS + \
NUM_D_SYMBOLS + \
NUM_LIT_SYMBOLS)
#define NUM_L_STATES (1 << 6)
#define NUM_M_STATES (1 << 6)
#define NUM_D_STATES (1 << 8)
#define NUM_LIT_STATES (1 << 10)
typedef UInt32 CFseState;
static UInt32 SumFreqs(const UInt16 *freqs, unsigned num)
{
UInt32 sum = 0;
for (unsigned i = 0; i < num; i++)
sum += (UInt32)freqs[i];
return sum;
}
static MY_FORCE_INLINE unsigned CountZeroBits(UInt32 val, UInt32 mask)
{
for (unsigned i = 0;;)
{
if (val & mask)
return i;
i++;
mask >>= 1;
}
}
static MY_FORCE_INLINE void InitLitTable(const UInt16 *freqs, UInt32 *table)
{
for (unsigned i = 0; i < NUM_LIT_SYMBOLS; i++)
{
unsigned f = freqs[i];
if (f == 0)
continue;
// 0 < f <= numStates
// 0 <= k <= numStatesLog
// numStates <= (f<<k) < numStates * 2
// 0 < j0 <= f
// (f + j0) = next_power_of_2 for f
unsigned k = CountZeroBits(f, NUM_LIT_STATES);
unsigned j0 = (((unsigned)NUM_LIT_STATES * 2) >> k) - f;
/*
CEntry
{
Byte k;
Byte symbol;
UInt16 delta;
};
*/
UInt32 e = ((UInt32)i << 8) + k;
k += 16;
UInt32 d = e + ((UInt32)f << k) - ((UInt32)NUM_LIT_STATES << 16);
UInt32 step = (UInt32)1 << k;
unsigned j = 0;
do
{
*table++ = d;
d += step;
}
while (++j < j0);
e--;
step >>= 1;
for (j = j0; j < f; j++)
{
*table++ = e;
e += step;
}
}
}
typedef struct
{
Byte totalBits;
Byte extraBits;
UInt16 delta;
UInt32 vbase;
} CExtraEntry;
static void InitExtraDecoderTable(unsigned numStates,
unsigned numSymbols,
const UInt16 *freqs,
const Byte *vbits,
CExtraEntry *table)
{
UInt32 vbase = 0;
for (unsigned i = 0; i < numSymbols; i++)
{
unsigned f = freqs[i];
unsigned extraBits = vbits[i];
if (f != 0)
{
unsigned k = CountZeroBits(f, numStates);
unsigned j0 = ((2 * numStates) >> k) - f;
unsigned j = 0;
do
{
CExtraEntry *e = table++;
e->totalBits = (Byte)(k + extraBits);
e->extraBits = (Byte)extraBits;
e->delta = (UInt16)(((f + j) << k) - numStates);
e->vbase = vbase;
}
while (++j < j0);
f -= j0;
k--;
for (j = 0; j < f; j++)
{
CExtraEntry *e = table++;
e->totalBits = (Byte)(k + extraBits);
e->extraBits = (Byte)extraBits;
e->delta = (UInt16)(j << k);
e->vbase = vbase;
}
}
vbase += ((UInt32)1 << extraBits);
}
}
static const Byte k_L_extra[NUM_L_SYMBOLS] =
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8
};
static const Byte k_M_extra[NUM_M_SYMBOLS] =
{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11
};
static const Byte k_D_extra[NUM_D_SYMBOLS] =
{
0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3,
4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11,
12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15
};
// ---------- CBitStream ----------
typedef struct
{
UInt32 accum;
unsigned numBits; // [0, 31] - Number of valid bits in (accum), other bits are 0
} CBitStream;
static MY_FORCE_INLINE int FseInStream_Init(CBitStream *s,
int n, // [-7, 0], (-n == number_of_unused_bits) in last byte
const Byte **pbuf)
{
*pbuf -= 4;
s->accum = GetUi32(*pbuf);
if (n)
{
s->numBits = n + 32;
if ((s->accum >> s->numBits) != 0)
return -1; // ERROR, encoder should have zeroed the upper bits
}
else
{
*pbuf += 1;
s->accum >>= 8;
s->numBits = 24;
}
return 0; // OK
}
// 0 <= numBits < 32
#define mask31(x, numBits) ((x) & (((UInt32)1 << (numBits)) - 1))
#define FseInStream_FLUSH \
{ unsigned nbits = (31 - in.numBits) & -8; \
if (nbits) { \
buf -= (nbits >> 3); \
if (buf < buf_check) return S_FALSE; \
UInt32 v = GetUi32(buf); \
in.accum = (in.accum << nbits) | mask31(v, nbits); \
in.numBits += nbits; }}
static MY_FORCE_INLINE UInt32 BitStream_Pull(CBitStream *s, unsigned numBits)
{
s->numBits -= numBits;
UInt32 v = s->accum >> s->numBits;
s->accum = mask31(s->accum, s->numBits);
return v;
}
#define DECODE_LIT(dest, pstate) { \
UInt32 e = lit_decoder[pstate]; \
pstate = (CFseState)((e >> 16) + BitStream_Pull(&in, e & 0xff)); \
dest = (Byte)(e >> 8); }
static MY_FORCE_INLINE UInt32 FseDecodeExtra(CFseState *pstate,
const CExtraEntry *table,
CBitStream *s)
{
const CExtraEntry *e = &table[*pstate];
UInt32 v = BitStream_Pull(s, e->totalBits);
unsigned extraBits = e->extraBits;
*pstate = (CFseState)(e->delta + (v >> extraBits));
return e->vbase + mask31(v, extraBits);
}
#define freqs_L (freqs)
#define freqs_M (freqs_L + NUM_L_SYMBOLS)
#define freqs_D (freqs_M + NUM_M_SYMBOLS)
#define freqs_LIT (freqs_D + NUM_D_SYMBOLS)
#define GET_BITS_64(v, offset, num, dest) dest = (UInt32) ((v >> (offset)) & ((1 << (num)) - 1));
#define GET_BITS_32(v, offset, num, dest) dest = (CFseState)((v >> (offset)) & ((1 << (num)) - 1));
HRESULT CDecoder::DecodeLzfse(UInt32 unpackSize, Byte version)
{
PRF(printf("\nLZFSE-%d %7u", version - '0', unpackSize));
UInt32 numLiterals;
UInt32 litPayloadSize;
Int32 literal_bits;
UInt32 lit_state_0;
UInt32 lit_state_1;
UInt32 lit_state_2;
UInt32 lit_state_3;
UInt32 numMatches;
UInt32 lmdPayloadSize;
Int32 lmd_bits;
CFseState l_state;
CFseState m_state;
CFseState d_state;
UInt16 freqs[NUM_SYMBOLS];
if (version == kSignature_LZFSE_V1)
{
return E_NOTIMPL;
// we need examples to test LZFSE-V1 code
/*
const unsigned k_v1_SubHeaderSize = 7 * 4 + 7 * 2;
const unsigned k_v1_HeaderSize = k_v1_SubHeaderSize + NUM_SYMBOLS * 2;
_buffer.AllocAtLeast(k_v1_HeaderSize);
if (m_InStream.ReadBytes(_buffer, k_v1_HeaderSize) != k_v1_HeaderSize)
return S_FALSE;
const Byte *buf = _buffer;
#define GET_32(offs, dest) dest = GetUi32(buf + offs)
#define GET_16(offs, dest) dest = GetUi16(buf + offs)
UInt32 payload_bytes;
GET_32(0, payload_bytes);
GET_32(4, numLiterals);
GET_32(8, numMatches);
GET_32(12, litPayloadSize);
GET_32(16, lmdPayloadSize);
if (litPayloadSize > (1 << 20) || lmdPayloadSize > (1 << 20))
return S_FALSE;
GET_32(20, literal_bits);
if (literal_bits < -7 || literal_bits > 0)
return S_FALSE;
GET_16(24, lit_state_0);
GET_16(26, lit_state_1);
GET_16(28, lit_state_2);
GET_16(30, lit_state_3);
GET_32(32, lmd_bits);
if (lmd_bits < -7 || lmd_bits > 0)
return S_FALSE;
GET_16(36, l_state);
GET_16(38, m_state);
GET_16(40, d_state);
for (unsigned i = 0; i < NUM_SYMBOLS; i++)
freqs[i] = GetUi16(buf + k_v1_SubHeaderSize + i * 2);
*/
}
else
{
UInt32 headerSize;
{
const unsigned kPreHeaderSize = 4 * 2; // signature and upackSize
const unsigned kHeaderSize = 8 * 3;
Byte temp[kHeaderSize];
if (m_InStream.ReadBytes(temp, kHeaderSize) != kHeaderSize)
return S_FALSE;
UInt64 v;
v = GetUi64(temp);
GET_BITS_64(v, 0, 20, numLiterals);
GET_BITS_64(v, 20, 20, litPayloadSize);
GET_BITS_64(v, 40, 20, numMatches);
GET_BITS_64(v, 60, 3 + 1, literal_bits); // (NumberOfUsedBits - 1)
literal_bits -= 7; // (-NumberOfUnusedBits)
if (literal_bits > 0)
return S_FALSE;
// GET_BITS_64(v, 63, 1, unused);
v = GetUi64(temp + 8);
GET_BITS_64(v, 0, 10, lit_state_0);
GET_BITS_64(v, 10, 10, lit_state_1);
GET_BITS_64(v, 20, 10, lit_state_2);
GET_BITS_64(v, 30, 10, lit_state_3);
GET_BITS_64(v, 40, 20, lmdPayloadSize);
GET_BITS_64(v, 60, 3 + 1, lmd_bits);
lmd_bits -= 7;
if (lmd_bits > 0)
return S_FALSE;
// GET_BITS_64(v, 63, 1, unused)
UInt32 v32 = GetUi32(temp + 20);
// (total header size in bytes; this does not
// correspond to a field in the uncompressed header version,
// but is required; we wouldn't know the size of the
// compresssed header otherwise.
GET_BITS_32(v32, 0, 10, l_state);
GET_BITS_32(v32, 10, 10, m_state);
GET_BITS_32(v32, 20, 10 + 2, d_state);
// GET_BITS_64(v, 62, 2, unused);
headerSize = GetUi32(temp + 16);
if (headerSize <= kPreHeaderSize + kHeaderSize)
return S_FALSE;
headerSize -= kPreHeaderSize + kHeaderSize;
}
// no freqs case is not allowed ?
// memset(freqs, 0, sizeof(freqs));
// if (headerSize != 0)
{
static const Byte numBitsTable[32] =
{
2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14,
2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14
};
static const Byte valueTable[32] =
{
0, 2, 1, 4, 0, 3, 1, 8, 0, 2, 1, 5, 0, 3, 1, 24,
0, 2, 1, 6, 0, 3, 1, 8, 0, 2, 1, 7, 0, 3, 1, 24
};
UInt32 accum = 0;
unsigned numBits = 0;
for (unsigned i = 0; i < NUM_SYMBOLS; i++)
{
while (numBits <= 14 && headerSize != 0)
{
Byte b;
if (!m_InStream.ReadByte(b))
return S_FALSE;
accum |= (UInt32)b << numBits;
numBits += 8;
headerSize--;
}
unsigned b = (unsigned)accum & 31;
unsigned n = numBitsTable[b];
if (numBits < n)
return S_FALSE;
numBits -= n;
UInt32 f = valueTable[b];
if (n >= 8)
f += ((accum >> 4) & (0x3ff >> (14 - n)));
accum >>= n;
freqs[i] = (UInt16)f;
}
if (numBits >= 8 || headerSize != 0)
return S_FALSE;
}
}
PRF(printf(" Literals=%6u Matches=%6u", numLiterals, numMatches));
if (numLiterals > LITERALS_PER_BLOCK
|| (numLiterals & 3) != 0
|| numMatches > MATCHES_PER_BLOCK
|| lit_state_0 >= NUM_LIT_STATES
|| lit_state_1 >= NUM_LIT_STATES
|| lit_state_2 >= NUM_LIT_STATES
|| lit_state_3 >= NUM_LIT_STATES
|| l_state >= NUM_L_STATES
|| m_state >= NUM_M_STATES
|| d_state >= NUM_D_STATES)
return S_FALSE;
// only full table is allowed ?
if ( SumFreqs(freqs_L, NUM_L_SYMBOLS) != NUM_L_STATES
|| SumFreqs(freqs_M, NUM_M_SYMBOLS) != NUM_M_STATES
|| SumFreqs(freqs_D, NUM_D_SYMBOLS) != NUM_D_STATES
|| SumFreqs(freqs_LIT, NUM_LIT_SYMBOLS) != NUM_LIT_STATES)
return S_FALSE;
const unsigned kPad = 16;
// ---------- Decode literals ----------
{
_literals.AllocAtLeast(LITERALS_PER_BLOCK + 16);
_buffer.AllocAtLeast(kPad + litPayloadSize);
memset(_buffer, 0, kPad);
if (m_InStream.ReadBytes(_buffer + kPad, litPayloadSize) != litPayloadSize)
return S_FALSE;
UInt32 lit_decoder[NUM_LIT_STATES];
InitLitTable(freqs_LIT, lit_decoder);
const Byte *buf_start = _buffer + kPad;
const Byte *buf_check = buf_start - 4;
const Byte *buf = buf_start + litPayloadSize;
CBitStream in;
if (FseInStream_Init(&in, literal_bits, &buf) != 0)
return S_FALSE;
Byte *lit = _literals;
const Byte *lit_limit = lit + numLiterals;
for (; lit < lit_limit; lit += 4)
{
FseInStream_FLUSH
DECODE_LIT (lit[0], lit_state_0);
DECODE_LIT (lit[1], lit_state_1);
FseInStream_FLUSH
DECODE_LIT (lit[2], lit_state_2);
DECODE_LIT (lit[3], lit_state_3);
}
if ((buf_start - buf) * 8 != (int)in.numBits)
return S_FALSE;
}
// ---------- Decode LMD ----------
_buffer.AllocAtLeast(kPad + lmdPayloadSize);
memset(_buffer, 0, kPad);
if (m_InStream.ReadBytes(_buffer + kPad, lmdPayloadSize) != lmdPayloadSize)
return S_FALSE;
CExtraEntry l_decoder[NUM_L_STATES];
CExtraEntry m_decoder[NUM_M_STATES];
CExtraEntry d_decoder[NUM_D_STATES];
InitExtraDecoderTable(NUM_L_STATES, NUM_L_SYMBOLS, freqs_L, k_L_extra, l_decoder);
InitExtraDecoderTable(NUM_M_STATES, NUM_M_SYMBOLS, freqs_M, k_M_extra, m_decoder);
InitExtraDecoderTable(NUM_D_STATES, NUM_D_SYMBOLS, freqs_D, k_D_extra, d_decoder);
const Byte *buf_start = _buffer + kPad;
const Byte *buf_check = buf_start - 4;
const Byte *buf = buf_start + lmdPayloadSize;
CBitStream in;
if (FseInStream_Init(&in, lmd_bits, &buf))
return S_FALSE;
const Byte *lit = _literals;
const Byte *lit_limit = lit + numLiterals;
UInt32 D = 0;
for (;;)
{
if (numMatches == 0)
break;
numMatches--;
FseInStream_FLUSH
unsigned L = (unsigned)FseDecodeExtra(&l_state, l_decoder, &in);
FseInStream_FLUSH
unsigned M = (unsigned)FseDecodeExtra(&m_state, m_decoder, &in);
FseInStream_FLUSH
{
UInt32 new_D = FseDecodeExtra(&d_state, d_decoder, &in);
if (new_D)
D = new_D;
}
if (L != 0)
{
if (L > (size_t)(lit_limit - lit))
return S_FALSE;
unsigned cur = L;
if (cur > unpackSize)
cur = (unsigned)unpackSize;
m_OutWindowStream.PutBytes(lit, cur);
unpackSize -= cur;
lit += cur;
if (cur != L)
return S_FALSE;
}
if (M != 0)
{
if (unpackSize == 0 || D == 0)
return S_FALSE;
unsigned cur = M;
if (cur > unpackSize)
cur = (unsigned)unpackSize;
if (!m_OutWindowStream.CopyBlock(D - 1, cur))
return S_FALSE;
unpackSize -= cur;
if (cur != M)
return S_FALSE;
}
}
if (unpackSize != 0)
return S_FALSE;
// LZFSE encoder writes 8 additional zero bytes before LMD payload
// We test it:
if ((buf - buf_start) * 8 + in.numBits != 64)
return S_FALSE;
if (GetUi64(buf_start) != 0)
return S_FALSE;
return S_OK;
}
STDMETHODIMP CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)
{
PRF(printf("\n\nLzfseDecoder %7u %7u\n", (unsigned)*outSize, (unsigned)*inSize));
const UInt32 kLzfseDictSize = 1 << 18;
if (!m_OutWindowStream.Create(kLzfseDictSize))
return E_OUTOFMEMORY;
if (!m_InStream.Create(1 << 18))
return E_OUTOFMEMORY;
m_OutWindowStream.SetStream(outStream);
m_OutWindowStream.Init(false);
m_InStream.SetStream(inStream);
m_InStream.Init();
CCoderReleaser coderReleaser(this);
UInt64 prevOut = 0;
UInt64 prevIn = 0;
for (;;)
{
const UInt64 pos = m_OutWindowStream.GetProcessedSize();
const UInt64 packPos = m_InStream.GetProcessedSize();
if (progress && ((pos - prevOut) >= (1 << 22) || (packPos - prevIn) >= (1 << 22)))
{
RINOK(progress->SetRatioInfo(&packPos, &pos));
prevIn = packPos;
prevOut = pos;
}
const UInt64 rem = *outSize - pos;
UInt32 v;
RINOK(GetUInt32(v))
if ((v & 0xFFFFFF) != 0x787662) // bvx
return S_FALSE;
v >>= 24;
if (v == 0x24) // '$', end of stream
break;
UInt32 unpackSize;
RINOK(GetUInt32(unpackSize));
UInt32 cur = unpackSize;
if (cur > rem)
cur = (UInt32)rem;
unpackSize -= cur;
HRESULT res;
if (v == kSignature_LZFSE_V1 || v == kSignature_LZFSE_V2)
res = DecodeLzfse(cur, (Byte)v);
else if (v == 0x6E) // 'n'
res = DecodeLzvn(cur);
else if (v == 0x2D) // '-'
res = DecodeUncompressed(cur);
else
return E_NOTIMPL;
if (res != S_OK)
return res;
if (unpackSize != 0)
return S_FALSE;
}
coderReleaser.NeedFlush = false;
HRESULT res = m_OutWindowStream.Flush();
if (res == S_OK)
if (*inSize != m_InStream.GetProcessedSize()
|| *outSize != m_OutWindowStream.GetProcessedSize())
res = S_FALSE;
return res;
}
STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)
{
try { return CodeReal(inStream, outStream, inSize, outSize, progress); }
catch(const CInBufferException &e) { return e.ErrorCode; }
catch(const CLzOutWindowException &e) { return e.ErrorCode; }
catch(...) { return E_OUTOFMEMORY; }
// catch(...) { return S_FALSE; }
}
}}

58
CPP/7zip/Compress/LzfseDecoder.h Normal file
View File

@@ -0,0 +1,58 @@
// LzfseDecoder.h
#ifndef __LZFSE_DECODER_H
#define __LZFSE_DECODER_H
#include "../../Common/MyBuffer.h"
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../Common/InBuffer.h"
#include "LzOutWindow.h"
namespace NCompress {
namespace NLzfse {
class CDecoder:
public ICompressCoder,
public CMyUnknownImp
{
CLzOutWindow m_OutWindowStream;
CInBuffer m_InStream;
CByteBuffer _literals;
CByteBuffer _buffer;
class CCoderReleaser
{
CDecoder *m_Coder;
public:
bool NeedFlush;
CCoderReleaser(CDecoder *coder): m_Coder(coder), NeedFlush(true) {}
~CCoderReleaser()
{
if (NeedFlush)
m_Coder->m_OutWindowStream.Flush();
}
};
friend class CCoderReleaser;
HRESULT GetUInt32(UInt32 &val);
HRESULT DecodeUncompressed(UInt32 unpackSize);
HRESULT DecodeLzvn(UInt32 unpackSize);
HRESULT DecodeLzfse(UInt32 unpackSize, Byte version);
STDMETHOD(CodeReal)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
public:
MY_UNKNOWN_IMP
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream, const UInt64 *inSize,
const UInt64 *outSize, ICompressProgressInfo *progress);
};
}}
#endif

53
CPP/7zip/Compress/Lzma2Encoder.cpp
View File

@@ -21,18 +21,19 @@ namespace NLzma2 {
CEncoder::CEncoder()
{
_encoder = 0;
_encoder = NULL;
_encoder = Lzma2Enc_Create(&g_Alloc, &g_BigAlloc);
if (_encoder == 0)
if (!_encoder)
throw 1;
}
CEncoder::~CEncoder()
{
if (_encoder != 0)
if (_encoder)
Lzma2Enc_Destroy(_encoder);
}
HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzma2Props)
{
switch (propID)
@@ -42,12 +43,7 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm
if (prop.vt == VT_UI4)
lzma2Props.blockSize = prop.ulVal;
else if (prop.vt == VT_UI8)
{
size_t v = (size_t)prop.uhVal.QuadPart;
if (v != prop.uhVal.QuadPart)
return E_INVALIDARG;
lzma2Props.blockSize = v;
}
lzma2Props.blockSize = prop.uhVal.QuadPart;
else
return E_INVALIDARG;
break;
@@ -60,6 +56,7 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm
return S_OK;
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
@@ -73,30 +70,52 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
return SResToHRESULT(Lzma2Enc_SetProps(_encoder, &lzma2Props));
}
STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT &prop = coderProps[i];
PROPID propID = propIDs[i];
if (propID == NCoderPropID::kExpectedDataSize)
if (prop.vt == VT_UI8)
Lzma2Enc_SetDataSize(_encoder, prop.uhVal.QuadPart);
}
return S_OK;
}
STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
{
Byte prop = Lzma2Enc_WriteProperties(_encoder);
return WriteStream(outStream, &prop, 1);
}
#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
{
CSeqInStreamWrap inWrap;
CSeqOutStreamWrap outWrap;
CCompressProgressWrap progressWrap;
inWrap.Init(inStream);
outWrap.Init(outStream);
progressWrap.Init(progress);
SRes res = Lzma2Enc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL);
if (res == SZ_ERROR_READ && inWrap.Res != S_OK)
return inWrap.Res;
if (res == SZ_ERROR_WRITE && outWrap.Res != S_OK)
return outWrap.Res;
if (res == SZ_ERROR_PROGRESS && progressWrap.Res != S_OK)
return progressWrap.Res;
SRes res = Lzma2Enc_Encode2(_encoder,
&outWrap.vt, NULL, NULL,
&inWrap.vt, NULL, 0,
progress ? &progressWrap.vt : NULL);
RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
return SResToHRESULT(res);
}

8
CPP/7zip/Compress/Lzma2Encoder.h
View File

@@ -16,16 +16,22 @@ class CEncoder:
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public ICompressSetCoderPropertiesOpt,
public CMyUnknownImp
{
CLzma2EncHandle _encoder;
public:
MY_UNKNOWN_IMP3(ICompressCoder, ICompressSetCoderProperties, ICompressWriteCoderProperties)
MY_UNKNOWN_IMP4(
ICompressCoder,
ICompressSetCoderProperties,
ICompressWriteCoderProperties,
ICompressSetCoderPropertiesOpt)
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
CEncoder();
virtual ~CEncoder();

39
CPP/7zip/Compress/LzmaEncoder.cpp
View File

@@ -74,14 +74,19 @@ HRESULT SetLzmaProp(PROPID propID, const PROPVARIANT &prop, CLzmaEncProps &ep)
return E_INVALIDARG;
return ParseMatchFinder(prop.bstrVal, &ep.btMode, &ep.numHashBytes) ? S_OK : E_INVALIDARG;
}
if (propID > NCoderPropID::kReduceSize)
return S_OK;
if (propID == NCoderPropID::kReduceSize)
{
if (prop.vt == VT_UI8)
ep.reduceSize = prop.uhVal.QuadPart;
else
return E_INVALIDARG;
return S_OK;
}
if (prop.vt != VT_UI4)
return E_INVALIDARG;
UInt32 v = prop.ulVal;
@@ -123,6 +128,22 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
return SResToHRESULT(LzmaEnc_SetProps(_encoder, &props));
}
STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
for (UInt32 i = 0; i < numProps; i++)
{
const PROPVARIANT &prop = coderProps[i];
PROPID propID = propIDs[i];
if (propID == NCoderPropID::kExpectedDataSize)
if (prop.vt == VT_UI8)
LzmaEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
}
return S_OK;
}
STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
{
Byte props[LZMA_PROPS_SIZE];
@@ -131,6 +152,10 @@ STDMETHODIMP CEncoder::WriteCoderProperties(ISequentialOutStream *outStream)
return WriteStream(outStream, props, size);
}
#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
{
@@ -142,15 +167,15 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream
outWrap.Init(outStream);
progressWrap.Init(progress);
SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL, &g_Alloc, &g_BigAlloc);
SRes res = LzmaEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt,
progress ? &progressWrap.vt : NULL, &g_Alloc, &g_BigAlloc);
_inputProcessed = inWrap.Processed;
if (res == SZ_ERROR_READ && inWrap.Res != S_OK)
return inWrap.Res;
if (res == SZ_ERROR_WRITE && outWrap.Res != S_OK)
return outWrap.Res;
if (res == SZ_ERROR_PROGRESS && progressWrap.Res != S_OK)
return progressWrap.Res;
RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
return SResToHRESULT(res);
}

8
CPP/7zip/Compress/LzmaEncoder.h
View File

@@ -16,17 +16,23 @@ class CEncoder:
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressWriteCoderProperties,
public ICompressSetCoderPropertiesOpt,
public CMyUnknownImp
{
CLzmaEncHandle _encoder;
UInt64 _inputProcessed;
public:
MY_UNKNOWN_IMP3(ICompressCoder, ICompressSetCoderProperties, ICompressWriteCoderProperties)
MY_UNKNOWN_IMP4(
ICompressCoder,
ICompressSetCoderProperties,
ICompressWriteCoderProperties,
ICompressSetCoderPropertiesOpt)
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD(WriteCoderProperties)(ISequentialOutStream *outStream);
STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
CEncoder();
virtual ~CEncoder();

13
CPP/7zip/Compress/Rar1Decoder.cpp
View File

@@ -29,7 +29,7 @@ public:
};
*/
CDecoder::CDecoder(): m_IsSolid(false) { }
CDecoder::CDecoder(): m_IsSolid(false), _errorMode(false) { }
void CDecoder::InitStructures()
{
@@ -406,9 +406,14 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
InitData();
if (!m_IsSolid)
{
_errorMode = false;
InitStructures();
InitHuff();
}
if (_errorMode)
return S_FALSE;
if (m_UnpackSize > 0)
{
GetFlagsBuf();
@@ -477,9 +482,9 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress)
{
try { return CodeReal(inStream, outStream, inSize, outSize, progress); }
catch(const CInBufferException &e) { return e.ErrorCode; }
catch(const CLzOutWindowException &e) { return e.ErrorCode; }
catch(...) { return S_FALSE; }
catch(const CInBufferException &e) { _errorMode = true; return e.ErrorCode; }
catch(const CLzOutWindowException &e) { _errorMode = true; return e.ErrorCode; }
catch(...) { _errorMode = true; return S_FALSE; }
}
STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte *data, UInt32 size)

1
CPP/7zip/Compress/Rar1Decoder.h
View File

@@ -39,6 +39,7 @@ public:
Int64 m_UnpackSize;
bool m_IsSolid;
bool _errorMode;
UInt32 ReadBits(int numBits);
HRESULT CopyBlock(UInt32 distance, UInt32 len);

23
CPP/7zip/Compress/Rar2Decoder.cpp
View File

@@ -130,7 +130,7 @@ bool CDecoder::ReadTables(void)
i = 0;
while (i < numLevels)
do
{
UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream);
if (sym < kTableDirectLevels)
@@ -144,10 +144,7 @@ bool CDecoder::ReadTables(void)
{
unsigned num = ReadBits(2) + 3;
if (i == 0)
{
// return false;
continue; // original unRAR
}
return false;
num += i;
if (num > numLevels)
{
@@ -180,6 +177,10 @@ bool CDecoder::ReadTables(void)
}
}
}
while (i < numLevels);
if (m_InBitStream.ExtraBitsWereRead())
return false;
if (m_AudioMode)
for (i = 0; i < m_NumChannels; i++)
@@ -244,6 +245,8 @@ bool CDecoder::DecodeMm(UInt32 pos)
while (pos-- != 0)
{
UInt32 symbol = m_MMDecoders[m_MmFilter.CurrentChannel].Decode(&m_InBitStream);
if (m_InBitStream.ExtraBitsWereRead())
return false;
if (symbol >= 256)
return symbol == 256;
/*
@@ -264,6 +267,8 @@ bool CDecoder::DecodeLz(Int32 pos)
while (pos > 0)
{
UInt32 sym = m_MainDecoder.Decode(&m_InBitStream);
if (m_InBitStream.ExtraBitsWereRead())
return false;
UInt32 length, distance;
if (sym < 256)
{
@@ -389,15 +394,19 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
if (!DecodeLz((Int32)blockSize))
return S_FALSE;
}
if (m_InBitStream.ExtraBitsWereRead())
return S_FALSE;
UInt64 globalPos = m_OutWindowStream.GetProcessedSize();
pos = globalPos - blockStartPos;
if (pos < blockSize)
if (!ReadTables())
return S_FALSE;
pos = globalPos - startPos;
if (progress != 0)
if (progress)
{
UInt64 packSize = m_InBitStream.GetProcessedSize();
const UInt64 packSize = m_InBitStream.GetProcessedSize();
RINOK(progress->SetRatioInfo(&packSize, &pos));
}
}

72
CPP/7zip/Compress/Rar3Decoder.cpp
View File

@@ -94,7 +94,8 @@ CDecoder::CDecoder():
_writtenFileSize(0),
_vmData(0),
_vmCode(0),
m_IsSolid(false)
m_IsSolid(false),
_errorMode(false)
{
Ppmd7_Construct(&_ppmd);
}
@@ -135,7 +136,7 @@ HRESULT CDecoder::WriteArea(UInt32 startPtr, UInt32 endPtr)
return WriteData(_window, endPtr);
}
void CDecoder::ExecuteFilter(int tempFilterIndex, NVm::CBlockRef &outBlockRef)
void CDecoder::ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockRef)
{
CTempFilter *tempFilter = _tempFilters[tempFilterIndex];
tempFilter->InitR[6] = (UInt32)_writtenFileSize;
@@ -144,7 +145,8 @@ void CDecoder::ExecuteFilter(int tempFilterIndex, NVm::CBlockRef &outBlockRef)
CFilter *filter = _filters[tempFilter->FilterIndex];
if (!filter->IsSupported)
_unsupportedFilter = true;
_vm.Execute(filter, tempFilter, outBlockRef, filter->GlobalData);
if (!_vm.Execute(filter, tempFilter, outBlockRef, filter->GlobalData))
_unsupportedFilter = true;
delete tempFilter;
_tempFilters[tempFilterIndex] = 0;
}
@@ -404,7 +406,7 @@ bool CDecoder::ReadVmCodePPM()
#define RIF(x) { if (!(x)) return S_FALSE; }
UInt32 CDecoder::ReadBits(int numBits) { return m_InBitStream.BitDecoder.ReadBits(numBits); }
UInt32 CDecoder::ReadBits(unsigned numBits) { return m_InBitStream.BitDecoder.ReadBits(numBits); }
// ---------- PPM ----------
@@ -413,7 +415,7 @@ HRESULT CDecoder::InitPPM()
unsigned maxOrder = (unsigned)ReadBits(7);
bool reset = ((maxOrder & 0x20) != 0);
int maxMB = 0;
UInt32 maxMB = 0;
if (reset)
maxMB = (Byte)ReadBits(8);
else
@@ -555,12 +557,13 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing)
PrevAlignCount = 0;
Byte levelLevels[kLevelTableSize];
Byte newLevels[kTablesSizesSum];
Byte lens[kTablesSizesSum];
if (ReadBits(1) == 0)
memset(m_LastLevels, 0, kTablesSizesSum);
int i;
unsigned i;
for (i = 0; i < kLevelTableSize; i++)
{
UInt32 length = ReadBits(4);
@@ -578,39 +581,44 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing)
}
levelLevels[i] = (Byte)length;
}
RIF(m_LevelDecoder.Build(levelLevels));
i = 0;
while (i < kTablesSizesSum)
do
{
UInt32 sym = m_LevelDecoder.Decode(&m_InBitStream.BitDecoder);
if (sym < 16)
{
newLevels[i] = Byte((sym + m_LastLevels[i]) & 15);
lens[i] = Byte((sym + m_LastLevels[i]) & 15);
i++;
}
else if (sym > kLevelTableSize)
return S_FALSE;
else
{
int num;
if (((sym - 16) & 1) == 0)
num = ReadBits(3) + 3;
else
num = ReadBits(7) + 11;
if (sym < 18)
unsigned num = ((sym - 16) & 1) * 4;
num += num + 3 + (unsigned)ReadBits(num + 3);
num += i;
if (num > kTablesSizesSum)
num = kTablesSizesSum;
Byte v = 0;
if (sym < 16 + 2)
{
if (i == 0)
return S_FALSE;
for (; num > 0 && i < kTablesSizesSum; num--, i++)
newLevels[i] = newLevels[(size_t)i - 1];
}
else
{
for (; num > 0 && i < kTablesSizesSum; num--)
newLevels[i++] = 0;
v = lens[(size_t)i - 1];
}
do
lens[i++] = v;
while (i < num);
}
}
while (i < kTablesSizesSum);
if (InputEofError())
return S_FALSE;
TablesRead = true;
@@ -623,12 +631,12 @@ HRESULT CDecoder::ReadTables(bool &keepDecompressing)
}
*/
RIF(m_MainDecoder.Build(&newLevels[0]));
RIF(m_DistDecoder.Build(&newLevels[kMainTableSize]));
RIF(m_AlignDecoder.Build(&newLevels[kMainTableSize + kDistTableSize]));
RIF(m_LenDecoder.Build(&newLevels[kMainTableSize + kDistTableSize + kAlignTableSize]));
RIF(m_MainDecoder.Build(&lens[0]));
RIF(m_DistDecoder.Build(&lens[kMainTableSize]));
RIF(m_AlignDecoder.Build(&lens[kMainTableSize + kDistTableSize]));
RIF(m_LenDecoder.Build(&lens[kMainTableSize + kDistTableSize + kAlignTableSize]));
memcpy(m_LastLevels, newLevels, kTablesSizesSum);
memcpy(m_LastLevels, lens, kTablesSizesSum);
TablesOK = true;
@@ -768,7 +776,7 @@ HRESULT CDecoder::DecodeLZ(bool &keepDecompressing)
if (sym2 >= kDistTableSize)
return S_FALSE;
rep0 = kDistStart[sym2];
int numBits = kDistDirectBits[sym2];
unsigned numBits = kDistDirectBits[sym2];
if (sym2 >= (kNumAlignBits * 2) + 2)
{
if (numBits > kNumAlignBits)
@@ -834,8 +842,12 @@ HRESULT CDecoder::CodeReal(ICompressProgressInfo *progress)
PpmEscChar = 2;
PpmError = true;
InitFilters();
_errorMode = false;
}
if (_errorMode)
return S_FALSE;
if (!m_IsSolid || !TablesRead)
{
bool keepDecompressing;
@@ -914,8 +926,8 @@ STDMETHODIMP CDecoder::Code(ISequentialInStream *inStream, ISequentialOutStream
_unpackSize = outSize ? *outSize : (UInt64)(Int64)-1;
return CodeReal(progress);
}
catch(const CInBufferException &e) { return e.ErrorCode; }
catch(...) { return S_FALSE; }
catch(const CInBufferException &e) { _errorMode = true; return e.ErrorCode; }
catch(...) { _errorMode = true; return S_FALSE; }
// CNewException is possible here. But probably CNewException is caused
// by error in data stream.
}

9
CPP/7zip/Compress/Rar3Decoder.h
View File

@@ -31,7 +31,7 @@ const UInt32 kLenTableSize = 28;
const UInt32 kMainTableSize = 256 + 1 + 1 + 1 + kNumReps + kNumLen2Symbols + kLenTableSize;
const UInt32 kDistTableSize = 60;
const int kNumAlignBits = 4;
const unsigned kNumAlignBits = 4;
const UInt32 kAlignTableSize = (1 << kNumAlignBits) + 1;
const UInt32 kLevelTableSize = 20;
@@ -158,7 +158,7 @@ struct CTempFilter: public NVm::CProgramInitState
}
};
const int kNumHuffmanBits = 15;
const unsigned kNumHuffmanBits = 15;
class CDecoder:
public ICompressCoder,
@@ -192,6 +192,7 @@ class CDecoder:
UInt32 _lastFilter;
bool m_IsSolid;
bool _errorMode;
bool _lzMode;
bool _unsupportedFilter;
@@ -209,7 +210,7 @@ class CDecoder:
HRESULT WriteDataToStream(const Byte *data, UInt32 size);
HRESULT WriteData(const Byte *data, UInt32 size);
HRESULT WriteArea(UInt32 startPtr, UInt32 endPtr);
void ExecuteFilter(int tempFilterIndex, NVm::CBlockRef &outBlockRef);
void ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockRef);
HRESULT WriteBuf();
void InitFilters();
@@ -217,7 +218,7 @@ class CDecoder:
bool ReadVmCodeLZ();
bool ReadVmCodePPM();
UInt32 ReadBits(int numBits);
UInt32 ReadBits(unsigned numBits);
HRESULT InitPPM();
int DecodePpmSymbol();

73
CPP/7zip/Compress/Rar3Vm.cpp
View File

@@ -155,7 +155,7 @@ bool CVm::Execute(CProgram *prg, const CProgramInitState *initState,
#ifdef RARVM_STANDARD_FILTERS
if (prg->StandardFilterIndex >= 0)
ExecuteStandardFilter(prg->StandardFilterIndex);
res = ExecuteStandardFilter(prg->StandardFilterIndex);
else
#endif
{
@@ -880,10 +880,10 @@ static void E8E9Decode(Byte *data, UInt32 dataSize, UInt32 fileOffset, bool e9)
if (((*data++) & cmpMask) == 0xE8)
{
UInt32 offset = curPos + fileOffset;
UInt32 addr = (Int32)GetValue32(data);
UInt32 addr = GetValue32(data);
if (addr < kFileSize)
SetValue32(data, addr - offset);
else if ((Int32)addr < 0 && (Int32)(addr + offset) >= 0)
else if ((addr & 0x80000000) != 0 && ((addr + offset) & 0x80000000) == 0)
SetValue32(data, addr + kFileSize);
data += 4;
curPos += 4;
@@ -935,7 +935,7 @@ static void ItaniumDecode(Byte *data, UInt32 dataSize, UInt32 fileOffset)
static void DeltaDecode(Byte *data, UInt32 dataSize, UInt32 numChannels)
{
UInt32 srcPos = 0;
UInt32 border = dataSize * 2;
const UInt32 border = dataSize * 2;
for (UInt32 curChannel = 0; curChannel < numChannels; curChannel++)
{
Byte prevByte = 0;
@@ -947,12 +947,13 @@ static void DeltaDecode(Byte *data, UInt32 dataSize, UInt32 numChannels)
static void RgbDecode(Byte *srcData, UInt32 dataSize, UInt32 width, UInt32 posR)
{
Byte *destData = srcData + dataSize;
const UInt32 numChannels = 3;
for (UInt32 curChannel = 0; curChannel < numChannels; curChannel++)
const UInt32 kNumChannels = 3;
for (UInt32 curChannel = 0; curChannel < kNumChannels; curChannel++)
{
Byte prevByte = 0;
for (UInt32 i = curChannel; i < dataSize; i+= numChannels)
for (UInt32 i = curChannel; i < dataSize; i += kNumChannels)
{
unsigned int predicted;
if (i < width)
@@ -978,7 +979,8 @@ static void RgbDecode(Byte *srcData, UInt32 dataSize, UInt32 width, UInt32 posR)
}
if (dataSize < 3)
return;
for (UInt32 i = posR, border = dataSize - 2; i < border; i += 3)
const UInt32 border = dataSize - 2;
for (UInt32 i = posR; i < border; i += 3)
{
Byte g = destData[i + 1];
destData[i ] = (Byte)(destData[i ] + g);
@@ -1064,11 +1066,11 @@ static UInt32 UpCaseDecode(Byte *data, UInt32 dataSize)
}
*/
void CVm::ExecuteStandardFilter(unsigned filterIndex)
bool CVm::ExecuteStandardFilter(unsigned filterIndex)
{
UInt32 dataSize = R[4];
if (dataSize >= kGlobalOffset)
return;
return false;
EStandardFilter filterType = kStdFilters[filterIndex].Type;
switch (filterType)
@@ -1077,42 +1079,59 @@ void CVm::ExecuteStandardFilter(unsigned filterIndex)
case SF_E8E9:
E8E9Decode(Mem, dataSize, R[6], (filterType == SF_E8E9));
break;
case SF_ITANIUM:
ItaniumDecode(Mem, dataSize, R[6]);
break;
case SF_DELTA:
{
if (dataSize >= kGlobalOffset / 2)
break;
return false;
UInt32 numChannels = R[0];
if (numChannels == 0 || numChannels > 1024) // unrar 5.5.5
return false;
SetBlockPos(dataSize);
DeltaDecode(Mem, dataSize, R[0]);
DeltaDecode(Mem, dataSize, numChannels);
break;
}
case SF_RGB:
if (dataSize >= kGlobalOffset / 2)
break;
{
UInt32 width = R[0];
if (width <= 3)
break;
SetBlockPos(dataSize);
RgbDecode(Mem, dataSize, width, R[1]);
}
break;
case SF_AUDIO:
if (dataSize >= kGlobalOffset / 2)
break;
{
if (dataSize >= kGlobalOffset / 2 || dataSize < 3) // unrar 5.5.5
return false;
UInt32 width = R[0];
UInt32 posR = R[1];
if (width < 3 || width - 3 > dataSize || posR > 2) // unrar 5.5.5
return false;
SetBlockPos(dataSize);
AudioDecode(Mem, dataSize, R[0]);
RgbDecode(Mem, dataSize, width, posR);
break;
}
case SF_AUDIO:
{
if (dataSize >= kGlobalOffset / 2)
return false;
UInt32 numChannels = R[0];
if (numChannels == 0 || numChannels > 128) // unrar 5.5.5
return false;
SetBlockPos(dataSize);
AudioDecode(Mem, dataSize, numChannels);
break;
}
/*
case SF_UPCASE:
if (dataSize >= kGlobalOffset / 2)
break;
return false;
UInt32 destSize = UpCaseDecode(Mem, dataSize);
SetBlockSize(destSize);
SetBlockPos(dataSize);
break;
*/
}
return true;
}
#endif

2
CPP/7zip/Compress/Rar3Vm.h
View File

@@ -173,7 +173,7 @@ private:
#endif
#ifdef RARVM_STANDARD_FILTERS
void ExecuteStandardFilter(unsigned filterIndex);
bool ExecuteStandardFilter(unsigned filterIndex);
#endif
Byte *Mem;

40
CPP/7zip/Compress/Rar5Decoder.cpp
View File

@@ -196,6 +196,8 @@ HRESULT CDecoder::ExecuteFilter(const CFilter &f)
default:
_unsupportedFilter = true;
memset(_filterSrc, 0, f.Size);
// return S_OK; // unrar
}
return WriteData(useDest ?
@@ -301,7 +303,11 @@ HRESULT CDecoder::AddFilter(CBitDecoder &_bitStream)
UInt32 blockStart = ReadUInt32(_bitStream);
f.Size = ReadUInt32(_bitStream);
// if (f.Size > ((UInt32)1 << 16)) _unsupportedFilter = true;
if (f.Size > ((UInt32)1 << 22))
{
_unsupportedFilter = true;
f.Size = 0; // unrar 5.5.5
}
f.Type = (Byte)_bitStream.ReadBits9fix(3);
f.Channels = 0;
@@ -415,7 +421,7 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream)
Byte lens[kTablesSizesSum];
unsigned i = 0;
while (i < kTablesSizesSum)
do
{
if (_bitStream._buf >= _bitStream._bufCheck2)
{
@@ -433,34 +439,24 @@ HRESULT CDecoder::ReadTables(CBitDecoder &_bitStream)
return S_FALSE;
else
{
sym -= 16;
unsigned sh = ((sym & 1) << 2);
unsigned num = (unsigned)_bitStream.ReadBits9(3 + sh) + 3 + (sh << 1);
unsigned num = ((sym - 16) & 1) * 4;
num += num + 3 + (unsigned)_bitStream.ReadBits9(num + 3);
num += i;
if (num > kTablesSizesSum)
num = kTablesSizesSum;
if (sym < 2)
Byte v = 0;
if (sym < 16 + 2)
{
if (i == 0)
{
// return S_FALSE;
continue; // original unRAR
}
Byte v = lens[(size_t)i - 1];
do
lens[i++] = v;
while (i < num);
}
else
{
do
lens[i++] = 0;
while (i < num);
return S_FALSE;
v = lens[(size_t)i - 1];
}
do
lens[i++] = v;
while (i < num);
}
}
while (i < kTablesSizesSum);
if (_bitStream.IsBlockOverRead())
return S_FALSE;

77
CPP/7zip/Compress/ShrinkDecoder.cpp
View File

@@ -13,6 +13,7 @@
namespace NCompress {
namespace NShrink {
static const UInt32 kEmpty = 256; // kNumItems;
static const UInt32 kBufferSize = (1 << 18);
static const unsigned kNumMinBits = 9;
@@ -34,20 +35,15 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
outBuffer.Init();
{
unsigned i;
for (i = 0; i < 257; i++)
_parents[i] = (UInt16)i;
for (; i < kNumItems; i++)
_parents[i] = kNumItems;
for (i = 0; i < kNumItems; i++)
_suffixes[i] = 0;
for (unsigned i = 0; i < kNumItems; i++)
_parents[i] = kEmpty;
}
UInt64 prevPos = 0, inPrev = 0;
UInt64 outPrev = 0, inPrev = 0;
unsigned numBits = kNumMinBits;
unsigned head = 257;
int lastSym = -1;
Byte lastChar2 = 0;
Byte lastChar = 0;
bool moreOut = false;
HRESULT res = S_FALSE;
@@ -67,18 +63,22 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
break;
}
eofCheck = true;
// Is specSym(=256) allowed after end of stream
// Do we need to read it here
// Is specSym(=256) allowed after end of stream ?
// Do we need to read it here ?
}
if (progress)
{
if (nowPos - prevPos >= (1 << 18)
|| _inProcessed - inPrev >= (1 << 20))
if (nowPos - outPrev >= (1 << 20) || _inProcessed - inPrev >= (1 << 20))
{
prevPos = nowPos;
outPrev = nowPos;
inPrev = _inProcessed;
RINOK(progress->SetRatioInfo(&_inProcessed, &nowPos));
res = progress->SetRatioInfo(&_inProcessed, &nowPos);
if (res != SZ_OK)
{
// break;
return res;
}
}
}
@@ -105,23 +105,30 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
continue;
}
if (sym != 2)
break;
{
break;
// continue; // info-zip just ignores such code
}
{
/*
---------- Free leaf nodes ----------
Note : that code can mark _parents[lastSym] as free, and next
inserted node will be Orphan in that case.
*/
unsigned i;
for (i = 257; i < kNumItems; i++)
for (i = 256; i < kNumItems; i++)
_stack[i] = 0;
for (i = 257; i < kNumItems; i++)
{
unsigned par = _parents[i];
if (par != kNumItems)
if (par != kEmpty)
_stack[par] = 1;
}
for (i = 257; i < kNumItems; i++)
if (_stack[i] == 0)
_parents[i] = kNumItems;
_parents[i] = kEmpty;
head = 257;
continue;
}
}
@@ -137,27 +144,22 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
bool needPrev = false;
if (head < kNumItems && lastSym >= 0)
{
while (head < kNumItems && _parents[head] != kNumItems)
while (head < kNumItems && _parents[head] != kEmpty)
head++;
if (head < kNumItems)
{
if (head == (unsigned)lastSym)
{
// we need to fix the code for that case
// _parents[head] is not allowed to link to itself
res = E_NOTIMPL;
break;
}
/*
if (head == lastSym), it updates Orphan to self-linked Orphan and creates two problems:
1) we must check _stack[i++] overflow in code that walks tree nodes.
2) self-linked node can not be removed. So such self-linked nodes can occupy all _parents items.
*/
needPrev = true;
_parents[head] = (UInt16)lastSym;
_suffixes[head] = (Byte)lastChar2;
_suffixes[head] = (Byte)lastChar;
head++;
}
}
if (_parents[sym] == kNumItems)
break;
lastSym = sym;
unsigned cur = sym;
unsigned i = 0;
@@ -166,10 +168,17 @@ HRESULT CDecoder::CodeReal(ISequentialInStream *inStream, ISequentialOutStream *
{
_stack[i++] = _suffixes[cur];
cur = _parents[cur];
// don't change that code:
// Orphan Check and self-linked Orphan check (_stack overflow check);
if (cur == kEmpty || i >= kNumItems)
break;
}
if (cur == kEmpty || i >= kNumItems)
break;
_stack[i++] = (Byte)cur;
lastChar2 = (Byte)cur;
lastChar = (Byte)cur;
if (needPrev)
_suffixes[(size_t)head - 1] = (Byte)cur;

2
CPP/7zip/Compress/ShrinkDecoder.h
View File

@@ -19,8 +19,8 @@ class CDecoder :
public ICompressGetInStreamProcessedSize,
public CMyUnknownImp
{
UInt64 _inProcessed;
bool _fullStreamMode;
UInt64 _inProcessed;
UInt16 _parents[kNumItems];
Byte _suffixes[kNumItems];

31
CPP/7zip/Compress/XzDecoder.cpp
View File

@@ -16,6 +16,19 @@ namespace NCompress {
namespace NXz {
CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(NULL), OutBuf(NULL)
{
XzUnpacker_Construct(&p, &g_Alloc);
}
CXzUnpackerCPP::~CXzUnpackerCPP()
{
XzUnpacker_Free(&p);
MidFree(InBuf);
MidFree(OutBuf);
}
void CStatInfo::Clear()
{
InSize = 0;
@@ -40,20 +53,6 @@ void CStatInfo::Clear()
}
CXzUnpackerCPP::CXzUnpackerCPP(): InBuf(0), OutBuf(0)
{
XzUnpacker_Construct(&p, &g_Alloc);
}
CXzUnpackerCPP::~CXzUnpackerCPP()
{
XzUnpacker_Free(&p);
MidFree(InBuf);
MidFree(OutBuf);
}
HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream *outStream,
const UInt64 *outSizeLimit, bool finishStream, ICompressProgressInfo *progress)
{
@@ -95,8 +94,12 @@ HRESULT CDecoder::Decode(ISequentialInStream *seqInStream, ISequentialOutStream
SizeT inLen = inSize - inPos;
SizeT outLen = kOutBufSize - outPos;
ECoderFinishMode finishMode = CODER_FINISH_ANY;
/*
// 17.01 : the code was disabled:
if (inSize == 0)
finishMode = CODER_FINISH_END;
*/
if (outSizeLimit)
{

6
CPP/7zip/Compress/XzDecoder.h
View File

@@ -3,13 +3,11 @@
#ifndef __XZ_DECODER_H
#define __XZ_DECODER_H
#include "../../Common/MyCom.h"
#include "../ICoder.h"
#include "../../../C/Xz.h"
#include "../../Common/MyCom.h"
// #include "../../Archive/XzHandler.h"
#include "../ICoder.h"
namespace NCompress {
namespace NXz {

321
CPP/7zip/Compress/XzEncoder.cpp
View File

@@ -4,6 +4,9 @@
#include "../../../C/Alloc.h"
#include "../../Common/MyString.h"
#include "../../Common/StringToInt.h"
#include "../Common/CWrappers.h"
#include "../Common/StreamUtils.h"
@@ -19,172 +22,224 @@ HRESULT SetLzma2Prop(PROPID propID, const PROPVARIANT &prop, CLzma2EncProps &lzm
namespace NXz {
extern "C" {
static void *SzBigAlloc(ISzAllocPtr, size_t size) { return BigAlloc(size); }
static void SzBigFree(ISzAllocPtr, void *address) { BigFree(address); }
static const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
static void *SzAlloc(ISzAllocPtr, size_t size) { return MyAlloc(size); }
static void SzFree(ISzAllocPtr, void *address) { MyFree(address); }
static const ISzAlloc g_Alloc = { SzAlloc, SzFree };
}
void CEncoder::InitCoderProps()
{
Lzma2EncProps_Init(&_lzma2Props);
XzProps_Init(&xzProps);
XzFilterProps_Init(&filter);
xzProps.lzma2Props = &_lzma2Props;
// xzProps.filterProps = (_filterId != 0 ? &filter : NULL);
xzProps.filterProps = NULL;
}
CEncoder::CEncoder()
{
InitCoderProps();
XzProps_Init(&xzProps);
_encoder = NULL;
_encoder = XzEnc_Create(&g_Alloc, &g_BigAlloc);
if (!_encoder)
throw 1;
}
CEncoder::~CEncoder()
{
if (_encoder)
XzEnc_Destroy(_encoder);
}
struct CMethodNamePair
{
UInt32 Id;
const char *Name;
};
static const CMethodNamePair g_NamePairs[] =
{
{ XZ_ID_Delta, "Delta" },
{ XZ_ID_X86, "BCJ" },
{ XZ_ID_PPC, "PPC" },
{ XZ_ID_IA64, "IA64" },
{ XZ_ID_ARM, "ARM" },
{ XZ_ID_ARMT, "ARMT" },
{ XZ_ID_SPARC, "SPARC" }
// { XZ_ID_LZMA2, "LZMA2" }
};
static int FilterIdFromName(const wchar_t *name)
{
for (unsigned i = 0; i < ARRAY_SIZE(g_NamePairs); i++)
{
const CMethodNamePair &pair = g_NamePairs[i];
if (StringsAreEqualNoCase_Ascii(name, pair.Name))
return (int)pair.Id;
}
return -1;
}
HRESULT CEncoder::SetCheckSize(UInt32 checkSizeInBytes)
{
unsigned id;
switch (checkSizeInBytes)
{
case 0: id = XZ_CHECK_NO; break;
case 4: id = XZ_CHECK_CRC32; break;
case 8: id = XZ_CHECK_CRC64; break;
case 32: id = XZ_CHECK_SHA256; break;
default: return E_INVALIDARG;
}
xzProps.checkId = id;
return S_OK;
}
HRESULT CEncoder::SetCoderProp(PROPID propID, const PROPVARIANT &prop)
{
return NLzma2::SetLzma2Prop(propID, prop, _lzma2Props);
if (propID == NCoderPropID::kNumThreads)
{
if (prop.vt != VT_UI4)
return E_INVALIDARG;
xzProps.numTotalThreads = (int)(prop.ulVal);
return S_OK;
}
if (propID == NCoderPropID::kCheckSize)
{
if (prop.vt != VT_UI4)
return E_INVALIDARG;
return SetCheckSize(prop.ulVal);
}
if (propID == NCoderPropID::kBlockSize2)
{
if (prop.vt == VT_UI4)
xzProps.blockSize = prop.ulVal;
else if (prop.vt == VT_UI8)
xzProps.blockSize = prop.uhVal.QuadPart;
else
return E_INVALIDARG;
return S_OK;
}
if (propID == NCoderPropID::kReduceSize)
{
if (prop.vt == VT_UI8)
xzProps.reduceSize = prop.uhVal.QuadPart;
else
return E_INVALIDARG;
return S_OK;
}
if (propID == NCoderPropID::kFilter)
{
if (prop.vt == VT_UI4)
{
UInt32 id32 = prop.ulVal;
if (id32 == XZ_ID_Delta)
return E_INVALIDARG;
xzProps.filterProps.id = prop.ulVal;
}
else
{
if (prop.vt != VT_BSTR)
return E_INVALIDARG;
const wchar_t *name = prop.bstrVal;
const wchar_t *end;
UInt32 id32 = ConvertStringToUInt32(name, &end);
if (end != name)
name = end;
else
{
if (IsString1PrefixedByString2_NoCase_Ascii(name, "Delta"))
{
name += 5; // strlen("Delta");
id32 = XZ_ID_Delta;
}
else
{
int filterId = FilterIdFromName(prop.bstrVal);
if (filterId < 0 /* || filterId == XZ_ID_LZMA2 */)
return E_INVALIDARG;
id32 = filterId;
}
}
if (id32 == XZ_ID_Delta)
{
wchar_t c = *name;
if (c != '-' && c != ':')
return E_INVALIDARG;
name++;
UInt32 delta = ConvertStringToUInt32(name, &end);
if (end == name || *end != 0 || delta == 0 || delta > 256)
return E_INVALIDARG;
xzProps.filterProps.delta = delta;
}
xzProps.filterProps.id = id32;
}
return S_OK;
}
return NLzma2::SetLzma2Prop(propID, prop, xzProps.lzma2Props);
}
STDMETHODIMP CEncoder::SetCoderProperties(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
Lzma2EncProps_Init(&_lzma2Props);
XzProps_Init(&xzProps);
for (UInt32 i = 0; i < numProps; i++)
{
RINOK(SetCoderProp(propIDs[i], coderProps[i]));
}
return S_OK;
// return SResToHRESULT(Lzma2Enc_SetProps(_encoder, &lzma2Props));
// return SResToHRESULT(XzEnc_SetProps(_encoder, &xzProps));
}
STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
ISequentialOutStream *outStream, const UInt64 * /* inSize */, const UInt64 * /* outSize */,
ICompressProgressInfo *progress)
STDMETHODIMP CEncoder::SetCoderPropertiesOpt(const PROPID *propIDs,
const PROPVARIANT *coderProps, UInt32 numProps)
{
CSeqOutStreamWrap seqOutStream;
seqOutStream.Init(outStream);
// if (IntToBool(newData))
for (UInt32 i = 0; i < numProps; i++)
{
/*
UInt64 size;
{
NCOM::CPropVariant prop;
RINOK(updateCallback->GetProperty(0, kpidSize, &prop));
if (prop.vt != VT_UI8)
return E_INVALIDARG;
size = prop.uhVal.QuadPart;
RINOK(updateCallback->SetTotal(size));
}
*/
/*
CLzma2EncProps lzma2Props;
Lzma2EncProps_Init(&lzma2Props);
lzma2Props.lzmaProps.level = GetLevel();
*/
CSeqInStreamWrap seqInStream;
seqInStream.Init(inStream);
/*
{
NCOM::CPropVariant prop = (UInt64)size;
RINOK(NCompress::NLzma2::SetLzma2Prop(NCoderPropID::kReduceSize, prop, lzma2Props));
}
FOR_VECTOR (i, _methods)
{
COneMethodInfo &m = _methods[i];
SetGlobalLevelAndThreads(m
#ifndef _7ZIP_ST
, _numThreads
#endif
);
{
FOR_VECTOR (j, m.Props)
{
const CProp &prop = m.Props[j];
RINOK(NCompress::NLzma2::SetLzma2Prop(prop.Id, prop.Value, lzma2Props));
}
}
}
#ifndef _7ZIP_ST
lzma2Props.numTotalThreads = _numThreads;
#endif
*/
CCompressProgressWrap progressWrap;
progressWrap.Init(progress);
xzProps.checkId = XZ_CHECK_CRC32;
// xzProps.checkId = XZ_CHECK_CRC64;
/*
CXzProps xzProps;
CXzFilterProps filter;
XzProps_Init(&xzProps);
XzFilterProps_Init(&filter);
xzProps.lzma2Props = &_lzma2Props;
*/
/*
xzProps.filterProps = (_filterId != 0 ? &filter : NULL);
switch (_crcSize)
{
case 0: xzProps.checkId = XZ_CHECK_NO; break;
case 4: xzProps.checkId = XZ_CHECK_CRC32; break;
case 8: xzProps.checkId = XZ_CHECK_CRC64; break;
case 32: xzProps.checkId = XZ_CHECK_SHA256; break;
default: return E_INVALIDARG;
}
filter.id = _filterId;
if (_filterId == XZ_ID_Delta)
{
bool deltaDefined = false;
FOR_VECTOR (j, _filterMethod.Props)
{
const CProp &prop = _filterMethod.Props[j];
if (prop.Id == NCoderPropID::kDefaultProp && prop.Value.vt == VT_UI4)
{
UInt32 delta = (UInt32)prop.Value.ulVal;
if (delta < 1 || delta > 256)
return E_INVALIDARG;
filter.delta = delta;
deltaDefined = true;
}
}
if (!deltaDefined)
return E_INVALIDARG;
}
*/
SRes res = Xz_Encode(&seqOutStream.vt, &seqInStream.vt, &xzProps, progress ? &progressWrap.vt : NULL);
/*
if (res == SZ_OK)
return updateCallback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK);
*/
return SResToHRESULT(res);
const PROPVARIANT &prop = coderProps[i];
PROPID propID = propIDs[i];
if (propID == NCoderPropID::kExpectedDataSize)
if (prop.vt == VT_UI8)
XzEnc_SetDataSize(_encoder, prop.uhVal.QuadPart);
}
return S_OK;
}
#define RET_IF_WRAP_ERROR(wrapRes, sRes, sResErrorCode) \
if (wrapRes != S_OK /* && (sRes == SZ_OK || sRes == sResErrorCode) */) return wrapRes;
STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 * /* inSize */, const UInt64 * /* outSize */, ICompressProgressInfo *progress)
{
CSeqInStreamWrap inWrap;
CSeqOutStreamWrap outWrap;
CCompressProgressWrap progressWrap;
inWrap.Init(inStream);
outWrap.Init(outStream);
progressWrap.Init(progress);
SRes res = XzEnc_SetProps(_encoder, &xzProps);
if (res == SZ_OK)
res = XzEnc_Encode(_encoder, &outWrap.vt, &inWrap.vt, progress ? &progressWrap.vt : NULL);
// SRes res = Xz_Encode(&outWrap.vt, &inWrap.vt, &xzProps, progress ? &progressWrap.vt : NULL);
RET_IF_WRAP_ERROR(inWrap.Res, res, SZ_ERROR_READ)
RET_IF_WRAP_ERROR(outWrap.Res, res, SZ_ERROR_WRITE)
RET_IF_WRAP_ERROR(progressWrap.Res, res, SZ_ERROR_PROGRESS)
return SResToHRESULT(res);
}
}}

14
CPP/7zip/Compress/XzEncoder.h
View File

@@ -16,24 +16,26 @@ namespace NXz {
class CEncoder:
public ICompressCoder,
public ICompressSetCoderProperties,
public ICompressSetCoderPropertiesOpt,
public CMyUnknownImp
{
// CXzEncHandle _encoder;
CXzEncHandle _encoder;
public:
CLzma2EncProps _lzma2Props;
CXzProps xzProps;
CXzFilterProps filter;
MY_UNKNOWN_IMP2(ICompressCoder, ICompressSetCoderProperties)
MY_UNKNOWN_IMP3(
ICompressCoder,
ICompressSetCoderProperties,
ICompressSetCoderPropertiesOpt)
void InitCoderProps();
HRESULT SetCheckSize(UInt32 checkSizeInBytes);
HRESULT SetCoderProp(PROPID propID, const PROPVARIANT &prop);
STDMETHOD(Code)(ISequentialInStream *inStream, ISequentialOutStream *outStream,
const UInt64 *inSize, const UInt64 *outSize, ICompressProgressInfo *progress);
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps);
CEncoder();
virtual ~CEncoder();

3
CPP/7zip/Crypto/Rar5Aes.cpp
View File

@@ -162,8 +162,9 @@ void CDecoder::Hmac_Convert_32Bytes(Byte *data) const
};
static CKey g_Key;
#ifndef _7ZIP_ST
static CKey g_Key;
static NWindows::NSynchronization::CCriticalSection g_GlobalKeyCacheCriticalSection;
#define MT_LOCK NWindows::NSynchronization::CCriticalSectionLock lock(g_GlobalKeyCacheCriticalSection);
#else

1
CPP/7zip/Guid.txt
View File

@@ -40,6 +40,7 @@
04 ICompressProgressInfo
05 ICompressCoder
18 ICompressCoder2
1F ICompressSetCoderPropertiesOpt
20 ICompressSetCoderProperties
21 ICompressSetDecoderProperties //
22 ICompressSetDecoderProperties2

47
CPP/7zip/ICoder.h
View File

@@ -104,25 +104,40 @@ namespace NCoderPropID
enum EEnum
{
kDefaultProp = 0,
kDictionarySize,
kUsedMemorySize,
kOrder,
kBlockSize,
kPosStateBits,
kLitContextBits,
kLitPosBits,
kNumFastBytes,
kMatchFinder,
kMatchFinderCycles,
kNumPasses,
kAlgorithm,
kNumThreads,
kEndMarker,
kLevel,
kReduceSize // estimated size of data that will be compressed. Encoder can use this value to reduce dictionary size.
kDictionarySize, // VT_UI4
kUsedMemorySize, // VT_UI4
kOrder, // VT_UI4
kBlockSize, // VT_UI4 or VT_UI8
kPosStateBits, // VT_UI4
kLitContextBits, // VT_UI4
kLitPosBits, // VT_UI4
kNumFastBytes, // VT_UI4
kMatchFinder, // VT_BSTR
kMatchFinderCycles, // VT_UI4
kNumPasses, // VT_UI4
kAlgorithm, // VT_UI4
kNumThreads, // VT_UI4
kEndMarker, // VT_BOOL
kLevel, // VT_UI4
kReduceSize, // VT_UI8 : it's estimated size of largest data stream that will be compressed
// encoder can use this value to reduce dictionary size and allocate data buffers
kExpectedDataSize, // VT_UI8 : for ICompressSetCoderPropertiesOpt :
// it's estimated size of current data stream
// real data size can differ from that size
// encoder can use this value to optimize encoder initialization
kBlockSize2, // VT_UI4 or VT_UI8
kCheckSize, // VT_UI4 : size of digest in bytes
kFilter // VT_BSTR
};
}
CODER_INTERFACE(ICompressSetCoderPropertiesOpt, 0x1F)
{
STDMETHOD(SetCoderPropertiesOpt)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) PURE;
};
CODER_INTERFACE(ICompressSetCoderProperties, 0x20)
{
STDMETHOD(SetCoderProperties)(const PROPID *propIDs, const PROPVARIANT *props, UInt32 numProps) PURE;

40
CPP/7zip/UI/Agent/Agent.cpp
View File

@@ -1217,7 +1217,12 @@ STDMETHODIMP CAgentFolder::GetFolderProperty(PROPID propID, PROPVARIANT *value)
NWindows::NCOM::CPropVariant prop;
if (propID == kpidReadOnly)
prop = _agentSpec->IsThereReadOnlyArc();
{
if (_agentSpec->Is_Attrib_ReadOnly())
prop = true;
else
prop = _agentSpec->IsThereReadOnlyArc();
}
else if (_proxy2)
{
const CProxyDir2 &dir = _proxy2->Dirs[_proxyDirIndex];
@@ -1453,7 +1458,12 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices,
pathMode = NExtract::NPathMode::kNoPathnames;
*/
extractCallbackSpec->InitForMulti(false, pathMode, overwriteMode);
extractCallbackSpec->InitForMulti(
false, // multiArchives
pathMode,
overwriteMode,
true // keepEmptyDirPrefixes
);
if (extractCallback2)
extractCallback2->SetTotal(_agentSpec->GetArc().GetEstmatedPhySize());
@@ -1502,11 +1512,18 @@ STDMETHODIMP CAgentFolder::Extract(const UInt32 *indices,
#endif
HRESULT result = _agentSpec->GetArchive()->Extract(&realIndices.Front(),
realIndices.Size(), testMode, extractCallback);
if (result == S_OK)
result = extractCallbackSpec->SetDirsTimes();
return result;
{
CArchiveExtractCallback_Closer ecsCloser(extractCallbackSpec);
HRESULT res = _agentSpec->GetArchive()->Extract(&realIndices.Front(),
realIndices.Size(), testMode, extractCallback);
HRESULT res2 = ecsCloser.Close();
if (res == S_OK)
res = res2;
return res;
}
COM_TRY_END
}
@@ -1552,6 +1569,7 @@ STDMETHODIMP CAgent::Open(
{
COM_TRY_BEGIN
_archiveFilePath = filePath;
_attrib = 0;
NFile::NFind::CFileInfo fi;
_isDeviceFile = false;
if (!inStream)
@@ -1560,6 +1578,7 @@ STDMETHODIMP CAgent::Open(
return ::GetLastError();
if (fi.IsDir())
return E_FAIL;
_attrib = fi.Attrib;
_isDeviceFile = fi.IsDevice;
}
CArcInfoEx archiverInfo0, archiverInfo1;
@@ -1717,7 +1736,12 @@ STDMETHODIMP CAgent::Extract(
COM_TRY_BEGIN
CArchiveExtractCallback *extractCallbackSpec = new CArchiveExtractCallback;
CMyComPtr<IArchiveExtractCallback> extractCallback = extractCallbackSpec;
extractCallbackSpec->InitForMulti(false, pathMode, overwriteMode);
extractCallbackSpec->InitForMulti(
false, // multiArchives
pathMode,
overwriteMode,
true // keepEmptyDirPrefixes
);
CExtractNtOptions extractNtOptions;
extractNtOptions.AltStreams.Val = true; // change it!!!

6
CPP/7zip/UI/Agent/Agent.h
View File

@@ -241,6 +241,7 @@ public:
CAgentFolder *_agentFolder;
UString _archiveFilePath;
DWORD _attrib;
bool _isDeviceFile;
#ifndef EXTRACT_ONLY
@@ -252,6 +253,11 @@ public:
IInArchive *GetArchive() const { if ( _archiveLink.Arcs.IsEmpty()) return 0; return GetArc().Archive; }
bool CanUpdate() const;
bool Is_Attrib_ReadOnly() const
{
return _attrib != INVALID_FILE_ATTRIBUTES && (_attrib & FILE_ATTRIBUTE_READONLY);
}
bool IsThereReadOnlyArc() const
{
FOR_VECTOR (i, _archiveLink.Arcs)

2
CPP/7zip/UI/Client7z/Client7z.cpp
View File

@@ -479,7 +479,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 operationResult)
}
_outFileStream.Release();
if (_extractMode && _processedFileInfo.AttribDefined)
SetFileAttrib(_diskFilePath, _processedFileInfo.Attrib);
SetFileAttrib_PosixHighDetect(_diskFilePath, _processedFileInfo.Attrib);
PrintNewLine();
return S_OK;
}

4
CPP/7zip/UI/Common/ArchiveCommandLine.cpp
View File

@@ -124,6 +124,7 @@ enum Enum
kTechMode,
kShareForWrite,
kStopAfterOpenError,
kCaseSensitive,
kArcNameMode,
@@ -245,6 +246,7 @@ static const CSwitchForm kSwitchForms[] =
{ "slt" },
{ "ssw" },
{ "sse" },
{ "ssc", NSwitchType::kMinus },
{ "sa", NSwitchType::kChar, false, 1, k_ArcNameMode_PostCharSet },
@@ -1299,6 +1301,8 @@ void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options)
if (parser[NKey::kShareForWrite].ThereIs)
updateOptions.OpenShareForWrite = true;
if (parser[NKey::kStopAfterOpenError].ThereIs)
updateOptions.StopAfterOpenError = true;
updateOptions.PathMode = censorPathMode;

183
CPP/7zip/UI/Common/ArchiveExtractCallback.cpp
View File

@@ -182,6 +182,7 @@ HRESULT CArchiveExtractCallback::PrepareHardLinks(const CRecordVector<UInt32> *r
#endif
CArchiveExtractCallback::CArchiveExtractCallback():
_arc(NULL),
WriteCTime(true),
WriteATime(true),
WriteMTime(true),
@@ -205,8 +206,8 @@ void CArchiveExtractCallback::Init(
const UStringVector &removePathParts, bool removePartsForAltStreams,
UInt64 packSize)
{
_extractedFolderPaths.Clear();
_extractedFolderIndices.Clear();
ClearExtractedDirsInfo();
_outFileStream.Release();
#ifdef SUPPORT_LINKS
_hardLinks.Clear();
@@ -368,9 +369,11 @@ void CArchiveExtractCallback::CreateComplexDirectory(const UStringVector &dirPat
}
}
HRESULT CArchiveExtractCallback::GetTime(int index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined)
HRESULT CArchiveExtractCallback::GetTime(UInt32 index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined)
{
filetimeIsDefined = false;
filetime.dwLowDateTime = 0;
filetime.dwHighDateTime = 0;
NCOM::CPropVariant prop;
RINOK(_arc->Archive->GetProperty(index, propID, &prop));
if (prop.vt == VT_FILETIME)
@@ -734,7 +737,8 @@ STDMETHODIMP CArchiveExtractCallback::GetStream(UInt32 index, ISequentialOutStre
return E_FAIL;
UString s;
CReparseAttr reparse;
isOkReparse = reparse.Parse((const Byte *)data, dataSize);
DWORD errorCode = 0;
isOkReparse = reparse.Parse((const Byte *)data, dataSize, errorCode);
if (isOkReparse)
{
isHardLink = false;
@@ -988,7 +992,7 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode)
|| !pathParts.IsEmpty()
|| !(_removePartsForAltStreams || _pathMode == NExtract::NPathMode::kNoPathsAlt))
#endif
Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, pathParts, _item.MainIsDir);
Correct_FsPath(_pathMode == NExtract::NPathMode::kAbsPaths, _keepAndReplaceEmptyDirPrefixes, pathParts, _item.MainIsDir);
#ifdef SUPPORT_ALT_STREAMS
@@ -1030,14 +1034,36 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode)
{
FString fullPathNew;
CreateComplexDirectory(pathParts, fullPathNew);
if (_item.IsDir)
{
_extractedFolderPaths.Add(fullPathNew);
_extractedFolderIndices.Add(index);
SetDirTime(fullPathNew,
(WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL,
(WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL,
(WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL));
CDirPathTime &pt = _extractedFolders.AddNew();
pt.CTime = _fi.CTime;
pt.CTimeDefined = (WriteCTime && _fi.CTimeDefined);
pt.ATime = _fi.ATime;
pt.ATimeDefined = (WriteATime && _fi.ATimeDefined);
pt.MTimeDefined = false;
if (WriteMTime)
{
if (_fi.MTimeDefined)
{
pt.MTime = _fi.MTime;
pt.MTimeDefined = true;
}
else if (_arc->MTimeDefined)
{
pt.MTime = _arc->MTime;
pt.MTimeDefined = true;
}
}
pt.Path = fullPathNew;
pt.SetDirTime();
}
}
}
@@ -1270,7 +1296,8 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode)
if (FillLinkData(data, fs2us(existPath), !isJunction))
{
CReparseAttr attr;
if (!attr.Parse(data, data.Size()))
DWORD errorCode = 0;
if (!attr.Parse(data, data.Size(), errorCode))
{
RINOK(SendMessageError("Internal error for symbolic link file", us2fs(_item.Path)));
// return E_FAIL;
@@ -1447,6 +1474,33 @@ STDMETHODIMP CArchiveExtractCallback::PrepareOperation(Int32 askExtractMode)
}
HRESULT CArchiveExtractCallback::CloseFile()
{
if (!_outFileStream)
return S_OK;
HRESULT hres = S_OK;
_outFileStreamSpec->SetTime(
(WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL,
(WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL,
(WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL));
const UInt64 processedSize = _outFileStreamSpec->ProcessedSize;
if (_fileLengthWasSet && _curSize > processedSize)
{
bool res = _outFileStreamSpec->File.SetLength(processedSize);
_fileLengthWasSet = res;
if (!res)
hres = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path));
}
_curSize = processedSize;
_curSizeDefined = true;
RINOK(_outFileStreamSpec->Close());
_outFileStream.Release();
return hres;
}
STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes)
{
COM_TRY_BEGIN
@@ -1475,27 +1529,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes)
#endif
if (_outFileStream)
{
HRESULT hres = S_OK;
_outFileStreamSpec->SetTime(
(WriteCTime && _fi.CTimeDefined) ? &_fi.CTime : NULL,
(WriteATime && _fi.ATimeDefined) ? &_fi.ATime : NULL,
(WriteMTime && _fi.MTimeDefined) ? &_fi.MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL));
const UInt64 processedSize = _outFileStreamSpec->ProcessedSize;
if (_fileLengthWasSet && _curSize > processedSize)
{
bool res = _outFileStreamSpec->File.SetLength(processedSize);
_fileLengthWasSet = res;
if (!res)
hres = SendMessageError_with_LastError(kCantSetFileLen, us2fs(_item.Path));
}
_curSize = processedSize;
_curSizeDefined = true;
RINOK(_outFileStreamSpec->Close());
_outFileStream.Release();
RINOK(hres);
}
RINOK(CloseFile());
#ifdef _USE_SECURITY_CODE
if (!_stdOutMode && _extractMode && _ntOptions.NtSecurity.Val && _arc->GetRawProps)
@@ -1542,7 +1576,7 @@ STDMETHODIMP CArchiveExtractCallback::SetOperationResult(Int32 opRes)
NumFiles++;
if (!_stdOutMode && _extractMode && _fi.AttribDefined)
SetFileAttrib(_diskFilePath, _fi.Attrib);
SetFileAttrib_PosixHighDetect(_diskFilePath, _fi.Attrib);
RINOK(_extractCallback2->SetOperationResult(opRes, BoolToInt(_encrypted)));
@@ -1592,71 +1626,66 @@ STDMETHODIMP CArchiveExtractCallback::CryptoGetTextPassword(BSTR *password)
}
struct CExtrRefSortPair
{
unsigned Len;
unsigned Index;
int Compare(const CExtrRefSortPair &a) const;
};
#define RINOZ(x) { int __tt = (x); if (__tt != 0) return __tt; }
int CExtrRefSortPair::Compare(const CExtrRefSortPair &a) const
{
RINOZ(-MyCompare(Len, a.Len));
return MyCompare(Index, a.Index);
}
static unsigned GetNumSlashes(const FChar *s)
void CDirPathSortPair::SetNumSlashes(const FChar *s)
{
for (unsigned numSlashes = 0;;)
{
FChar c = *s++;
if (c == 0)
return numSlashes;
{
Len = numSlashes;
return;
}
if (IS_PATH_SEPAR(c))
numSlashes++;
}
}
bool CDirPathTime::SetDirTime()
{
return NDir::SetDirTime(Path,
CTimeDefined ? &CTime : NULL,
ATimeDefined ? &ATime : NULL,
MTimeDefined ? &MTime : NULL);
}
HRESULT CArchiveExtractCallback::SetDirsTimes()
{
CRecordVector<CExtrRefSortPair> pairs;
pairs.ClearAndSetSize(_extractedFolderPaths.Size());
if (!_arc)
return S_OK;
CRecordVector<CDirPathSortPair> pairs;
pairs.ClearAndSetSize(_extractedFolders.Size());
unsigned i;
for (i = 0; i < _extractedFolderPaths.Size(); i++)
for (i = 0; i < _extractedFolders.Size(); i++)
{
CExtrRefSortPair &pair = pairs[i];
CDirPathSortPair &pair = pairs[i];
pair.Index = i;
pair.Len = GetNumSlashes(_extractedFolderPaths[i]);
pair.SetNumSlashes(_extractedFolders[i].Path);
}
pairs.Sort2();
for (i = 0; i < pairs.Size(); i++)
{
int pairIndex = pairs[i].Index;
int index = _extractedFolderIndices[pairIndex];
FILETIME CTime;
FILETIME ATime;
FILETIME MTime;
bool CTimeDefined;
bool ATimeDefined;
bool MTimeDefined;
RINOK(GetTime(index, kpidCTime, CTime, CTimeDefined));
RINOK(GetTime(index, kpidATime, ATime, ATimeDefined));
RINOK(GetTime(index, kpidMTime, MTime, MTimeDefined));
// printf("\n%S", _extractedFolderPaths[pairIndex]);
SetDirTime(_extractedFolderPaths[pairIndex],
(WriteCTime && CTimeDefined) ? &CTime : NULL,
(WriteATime && ATimeDefined) ? &ATime : NULL,
(WriteMTime && MTimeDefined) ? &MTime : (_arc->MTimeDefined ? &_arc->MTime : NULL));
_extractedFolders[pairs[i].Index].SetDirTime();
// if (!) return GetLastError();
}
ClearExtractedDirsInfo();
return S_OK;
}
HRESULT CArchiveExtractCallback::CloseArc()
{
HRESULT res = CloseFile();
HRESULT res2 = SetDirsTimes();
if (res == S_OK)
res = res2;
_arc = NULL;
return res;
}

63
CPP/7zip/UI/Common/ArchiveExtractCallback.h
View File

@@ -151,6 +151,25 @@ struct CIndexToPathPair
#endif
struct CDirPathTime
{
FILETIME CTime;
FILETIME ATime;
FILETIME MTime;
bool CTimeDefined;
bool ATimeDefined;
bool MTimeDefined;
FString Path;
bool SetDirTime();
};
class CArchiveExtractCallback:
public IArchiveExtractCallback,
public IArchiveExtractCallbackMessage,
@@ -172,6 +191,7 @@ class CArchiveExtractCallback:
FString _dirPathPrefix_Full;
NExtract::NPathMode::EEnum _pathMode;
NExtract::NOverwriteMode::EEnum _overwriteMode;
bool _keepAndReplaceEmptyDirPrefixes; // replace them to "_";
#ifndef _SFX
@@ -240,15 +260,14 @@ class CArchiveExtractCallback:
UInt64 _progressTotal;
bool _progressTotal_Defined;
FStringVector _extractedFolderPaths;
CRecordVector<UInt32> _extractedFolderIndices;
CObjectVector<CDirPathTime> _extractedFolders;
#if defined(_WIN32) && !defined(UNDER_CE) && !defined(_SFX)
bool _saclEnabled;
#endif
void CreateComplexDirectory(const UStringVector &dirPathParts, FString &fullPath);
HRESULT GetTime(int index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined);
HRESULT GetTime(UInt32 index, PROPID propID, FILETIME &filetime, bool &filetimeIsDefined);
HRESULT GetUnpackSize();
HRESULT SendMessageError(const char *message, const FString &path);
@@ -278,11 +297,13 @@ public:
void InitForMulti(bool multiArchives,
NExtract::NPathMode::EEnum pathMode,
NExtract::NOverwriteMode::EEnum overwriteMode)
NExtract::NOverwriteMode::EEnum overwriteMode,
bool keepAndReplaceEmptyDirPrefixes)
{
_multiArchives = multiArchives;
_pathMode = pathMode;
_overwriteMode = overwriteMode;
_keepAndReplaceEmptyDirPrefixes = keepAndReplaceEmptyDirPrefixes;
NumFolders = NumFiles = NumAltStreams = UnpackSize = AltStreams_UnpackSize = 0;
}
@@ -340,9 +361,43 @@ public:
}
#endif
HRESULT CloseArc();
private:
void ClearExtractedDirsInfo()
{
_extractedFolders.Clear();
}
HRESULT CloseFile();
HRESULT SetDirsTimes();
};
struct CArchiveExtractCallback_Closer
{
CArchiveExtractCallback *_ref;
CArchiveExtractCallback_Closer(CArchiveExtractCallback *ref): _ref(ref) {}
HRESULT Close()
{
HRESULT res = S_OK;
if (_ref)
{
res = _ref->CloseArc();
_ref = NULL;
}
return res;
}
~CArchiveExtractCallback_Closer()
{
Close();
}
};
bool CensorNode_CheckPath(const NWildcard::CCensorNode &node, const CReadArcItem &item);
#endif

113
CPP/7zip/UI/Common/Bench.cpp
View File

@@ -2376,15 +2376,7 @@ static UInt32 GetNumThreadsNext(unsigned i, UInt32 numThreads)
static bool AreSameMethodNames(const char *fullName, const char *shortName)
{
for (;;)
{
char c2 = *shortName++;
if (c2 == 0)
return true;
char c1 = *fullName++;
if (MyCharLower_Ascii(c1) != MyCharLower_Ascii(c2))
return false;
}
return StringsAreEqualNoCase_Ascii(fullName, shortName);
}
@@ -2545,27 +2537,24 @@ static const char * const k_PF[] =
#endif
static void PrintSize(AString &s, UInt64 ptr)
static void PrintSize(AString &s, UInt64 v)
{
UInt64 v = ptr;
UInt64 t = v >> 40;
UInt64 g = v >> 30;
UInt64 m = v >> 20;
UInt64 k = v >> 10;
UInt32 d = (UInt32)v;
char c = 0;
if (t >= 1000) { d = (UInt32)t; c = 'T'; }
else if (g >= 1000) { d = (UInt32)g; c = 'G'; }
else if (m >= 1000) { d = (UInt32)m; c = 'M'; }
else if (k >= 10) { d = (UInt32)k; c = 'K'; }
s.Add_UInt32(d);
// s += ' ';
if ((v & 0x3FF) == 0) { v >>= 10; c = 'K';
if ((v & 0x3FF) == 0) { v >>= 10; c = 'M';
if ((v & 0x3FF) == 0) { v >>= 10; c = 'G';
if ((v & 0x3FF) == 0) { v >>= 10; c = 'T';
}}}}
else
{
PrintHex(s, v);
return;
}
char temp[32];
ConvertUInt64ToString(v, temp);
s += temp;
if (c)
s += c;
// PrintHex(s, (DWORD_PTR)v);
}
@@ -2631,12 +2620,19 @@ static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si)
s += " ";
DWORD_PTR minAdd = (DWORD_PTR)si.lpMinimumApplicationAddress;
if (minAdd != (1 << 16))
UInt64 maxSize = (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1;
const UInt32 kReserveSize = ((UInt32)1 << 16);
if (minAdd != kReserveSize)
{
PrintSize(s, minAdd);
s += "-";
}
PrintSize(s, (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1);
else
{
if ((maxSize & (kReserveSize - 1)) == 0)
maxSize += kReserveSize;
}
PrintSize(s, maxSize);
}
#ifndef _WIN64
@@ -3006,26 +3002,34 @@ HRESULT Bench(
UInt32 complexity = 10000;
const UInt32 *checkSum = NULL;
{
for (unsigned i = 0; i < ARRAY_SIZE(g_Hash); i++)
unsigned i;
for (i = 0; i < ARRAY_SIZE(g_Hash); i++)
{
const CBenchHash &h = g_Hash[i];
AString s (h.Name);
AString hProp;
int propPos = s.Find(':');
AString benchMethod (h.Name);
AString benchProps;
int propPos = benchMethod.Find(':');
if (propPos >= 0)
{
hProp = s.Ptr(propPos + 1);
s.DeleteFrom(propPos);
benchProps = benchMethod.Ptr(propPos + 1);
benchMethod.DeleteFrom(propPos);
}
if (AreSameMethodNames(s, methodName))
if (AreSameMethodNames(benchMethod, methodName))
{
complexity = h.Complex;
checkSum = &h.CheckSum;
if (method.PropsString.IsEqualTo_Ascii_NoCase(hProp))
break;
if (benchProps.IsEmpty()
|| benchMethod.IsEqualTo_Ascii_NoCase("crc32") && benchProps == "8" && method.PropsString.IsEmpty()
|| method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps))
{
complexity = h.Complex;
checkSum = &h.CheckSum;
if (method.PropsString.IsEqualTo_Ascii_NoCase(benchProps))
break;
}
}
}
if (i == ARRAY_SIZE(g_Hash))
return E_NOTIMPL;
}
f.NewLine();
@@ -3316,18 +3320,35 @@ HRESULT Bench(
bool needSetComplexity = true;
if (!methodName.IsEqualTo_Ascii_NoCase("LZMA"))
{
for (unsigned i = 0; i < ARRAY_SIZE(g_Bench); i++)
unsigned i;
for (i = 0; i < ARRAY_SIZE(g_Bench); i++)
{
const CBenchMethod &h = g_Bench[i];
if (AreSameMethodNames(h.Name, methodName))
AString benchMethod (h.Name);
AString benchProps;
int propPos = benchMethod.Find(':');
if (propPos >= 0)
{
callback.BenchProps.EncComplex = h.EncComplex;
callback.BenchProps.DecComplexCompr = h.DecComplexCompr;
callback.BenchProps.DecComplexUnc = h.DecComplexUnc;;
needSetComplexity = false;
break;
benchProps = benchMethod.Ptr(propPos + 1);
benchMethod.DeleteFrom(propPos);
}
if (AreSameMethodNames(benchMethod, methodName))
{
if (benchProps.IsEmpty()
|| benchProps == "x5" && method.PropsString.IsEmpty()
|| method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps))
{
callback.BenchProps.EncComplex = h.EncComplex;
callback.BenchProps.DecComplexCompr = h.DecComplexCompr;
callback.BenchProps.DecComplexUnc = h.DecComplexUnc;;
needSetComplexity = false;
break;
}
}
}
if (i == ARRAY_SIZE(g_Bench))
return E_NOTIMPL;
}
if (needSetComplexity)
callback.BenchProps.SetLzmaCompexity();

6
CPP/7zip/UI/Common/CompressCall.cpp
View File

@@ -39,7 +39,6 @@ using namespace NWindows;
#define kArcIncludeSwitches " -an -ai"
#define kHashIncludeSwitches " -i"
#define kStopSwitchParsing " --"
#define kLargePagesDisable " -slp-"
extern HWND g_HWND;
@@ -94,8 +93,8 @@ static HRESULT Call7zGui(const UString &params,
static void AddLagePagesSwitch(UString &params)
{
if (!ReadLockMemoryEnable())
params += kLargePagesDisable;
if (ReadLockMemoryEnable())
params += " -slp";
}
class CRandNameGenerator
@@ -289,6 +288,7 @@ void Benchmark(bool totalMode)
UString params ('b');
if (totalMode)
params += " -mm=*";
AddLagePagesSwitch(params);
HRESULT result = Call7zGui(params, false, NULL);
if (result != S_OK)
ErrorMessageHRESULT(result);

26
CPP/7zip/UI/Common/DirItem.h
View File

@@ -21,7 +21,7 @@ struct CDirItemsStat
UInt64 NumErrors;
UInt64 Get_NumItems() const { return NumDirs + NumFiles + NumAltStreams; }
// UInt64 Get_NumItems() const { return NumDirs + NumFiles + NumAltStreams; }
UInt64 Get_NumDataItems() const { return NumFiles + NumAltStreams; }
UInt64 GetTotalBytes() const { return FilesSize + AltStreamsSize; }
@@ -43,6 +43,30 @@ struct CDirItemsStat
{}
};
struct CDirItemsStat2: public CDirItemsStat
{
UInt64 Anti_NumDirs;
UInt64 Anti_NumFiles;
UInt64 Anti_NumAltStreams;
// UInt64 Get_NumItems() const { return Anti_NumDirs + Anti_NumFiles + Anti_NumAltStreams + CDirItemsStat::Get_NumItems(); }
UInt64 Get_NumDataItems2() const { return Anti_NumFiles + Anti_NumAltStreams + CDirItemsStat::Get_NumDataItems(); }
bool IsEmpty() const { return CDirItemsStat::IsEmpty()
&& 0 == Anti_NumDirs
&& 0 == Anti_NumFiles
&& 0 == Anti_NumAltStreams; }
CDirItemsStat2():
Anti_NumDirs(0),
Anti_NumFiles(0),
Anti_NumAltStreams(0)
{}
};
#define INTERFACE_IDirItemsCallback(x) \
virtual HRESULT ScanError(const FString &path, DWORD systemError) x; \
virtual HRESULT ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) x; \

38
CPP/7zip/UI/Common/EnumDirItems.cpp
View File

@@ -343,6 +343,7 @@ static HRESULT EnumerateAltStreams(
const NWildcard::CCensorNode &curNode,
int phyParent, int logParent, const FString &fullPath,
const UStringVector &addArchivePrefix, // prefix from curNode
bool addAllItems,
CDirItems &dirItems)
{
NFind::CStreamEnumerator enumerator(fullPath);
@@ -363,6 +364,10 @@ static HRESULT EnumerateAltStreams(
addArchivePrefixNew.Back() += reducedName;
if (curNode.CheckPathToRoot(false, addArchivePrefixNew, true))
continue;
if (!addAllItems)
if (!curNode.CheckPathToRoot(true, addArchivePrefixNew, true))
continue;
NFind::CFileInfo fi2 = fi;
fi2.Name += us2fs(reducedName);
fi2.Size = si.Size;
@@ -381,15 +386,27 @@ HRESULT CDirItems::SetLinkInfo(CDirItem &dirItem, const NFind::CFileInfo &fi,
return S_OK;
const FString path = phyPrefix + fi.Name;
CByteBuffer &buf = dirItem.ReparseData;
DWORD res = 0;
if (NIO::GetReparseData(path, buf))
{
CReparseAttr attr;
if (attr.Parse(buf, buf.Size()))
if (attr.Parse(buf, buf.Size(), res))
return S_OK;
// we ignore unknown reparse points
if (res != ERROR_INVALID_REPARSE_DATA)
res = 0;
}
DWORD res = ::GetLastError();
else
{
res = ::GetLastError();
if (res == 0)
res = ERROR_INVALID_FUNCTION;
}
buf.Free();
return AddError(path , res);
if (res == 0)
return S_OK;
return AddError(path, res);
}
#endif
@@ -413,6 +430,8 @@ static HRESULT EnumerateForItem(
}
int dirItemIndex = -1;
bool addAllSubStreams = false;
if (curNode.CheckPathToRoot(true, addArchivePrefixNew, !fi.IsDir()))
{
int secureIndex = -1;
@@ -427,6 +446,8 @@ static HRESULT EnumerateForItem(
dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi);
if (fi.IsDir())
enterToSubFolders2 = true;
addAllSubStreams = true;
}
#ifndef UNDER_CE
@@ -434,7 +455,9 @@ static HRESULT EnumerateForItem(
{
RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent,
phyPrefix + fi.Name,
addArchivePrefixNew, dirItems));
addArchivePrefixNew,
addAllSubStreams,
dirItems));
}
if (dirItemIndex >= 0)
@@ -643,7 +666,9 @@ static HRESULT EnumerateDirItems(
UStringVector pathParts;
pathParts.Add(fs2us(fi.Name));
RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent,
fullPath, pathParts, dirItems));
fullPath, pathParts,
true, /* addAllSubStreams */
dirItems));
}
#endif
@@ -852,7 +877,8 @@ void CDirItems::FillFixedReparse()
continue;
CReparseAttr attr;
if (!attr.Parse(item.ReparseData, item.ReparseData.Size()))
DWORD errorCode = 0;
if (!attr.Parse(item.ReparseData, item.ReparseData.Size(), errorCode))
continue;
if (attr.IsRelative())
continue;

14
CPP/7zip/UI/Common/Extract.cpp
View File

@@ -197,6 +197,9 @@ static HRESULT DecompressArchive(
HRESULT result;
Int32 testMode = (options.TestMode && !calcCrc) ? 1: 0;
CArchiveExtractCallback_Closer ecsCloser(ecs);
if (options.StdInMode)
{
result = archive->Extract(NULL, (UInt32)(Int32)-1, testMode, ecs);
@@ -206,8 +209,11 @@ static HRESULT DecompressArchive(
}
else
result = archive->Extract(&realIndices.Front(), realIndices.Size(), testMode, ecs);
if (result == S_OK && !options.StdInMode)
result = ecs->SetDirsTimes();
HRESULT res2 = ecsCloser.Close();
if (result == S_OK)
result = res2;
return callback->ExtractResult(result);
}
@@ -279,7 +285,9 @@ HRESULT Extract(
CArchiveExtractCallback *ecs = new CArchiveExtractCallback;
CMyComPtr<IArchiveExtractCallback> ec(ecs);
bool multi = (numArcs > 1);
ecs->InitForMulti(multi, options.PathMode, options.OverwriteMode);
ecs->InitForMulti(multi, options.PathMode, options.OverwriteMode,
false // keepEmptyDirParts
);
#ifndef _SFX
ecs->SetHashMethods(hash);
#endif

23
CPP/7zip/UI/Common/ExtractingFilePath.cpp
View File

@@ -146,6 +146,9 @@ static void CorrectUnsupportedName(UString &name)
static void Correct_PathPart(UString &s)
{
// "." and ".."
if (s.IsEmpty())
return;
if (s[0] == '.' && (s[1] == 0 || s[1] == '.' && s[2] == 0))
s.Empty();
#ifdef _WIN32
@@ -172,7 +175,7 @@ UString Get_Correct_FsFile_Name(const UString &name)
}
void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir)
void Correct_FsPath(bool absIsAllowed, bool keepAndReplaceEmptyPrefixes, UStringVector &parts, bool isDir)
{
unsigned i = 0;
@@ -181,6 +184,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir)
#if defined(_WIN32) && !defined(UNDER_CE)
bool isDrive = false;
#endif
if (parts[0].IsEmpty())
{
i = 1;
@@ -191,7 +195,7 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir)
if (parts.Size() > 2 && parts[2] == L"?")
{
i = 3;
if (parts.Size() > 3 && NWindows::NFile::NName::IsDrivePath2(parts[3]))
if (parts.Size() > 3 && NWindows::NFile::NName::IsDrivePath2(parts[3]))
{
isDrive = true;
i = 4;
@@ -220,6 +224,9 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir)
#endif
}
if (i != 0)
keepAndReplaceEmptyPrefixes = false;
for (; i < parts.Size();)
{
UString &s = parts[i];
@@ -228,15 +235,17 @@ void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir)
if (s.IsEmpty())
{
if (isDir || i != parts.Size() - 1)
{
parts.Delete(i);
continue;
}
if (!keepAndReplaceEmptyPrefixes)
if (isDir || i != parts.Size() - 1)
{
parts.Delete(i);
continue;
}
s = k_EmptyReplaceName;
}
else
{
keepAndReplaceEmptyPrefixes = false;
#ifdef _WIN32
CorrectUnsupportedName(s);
#endif

14
CPP/7zip/UI/Common/ExtractingFilePath.h
View File

@@ -12,7 +12,19 @@ void Correct_AltStream_Name(UString &s);
// replaces unsuported characters, and replaces "." , ".." and "" to "[]"
UString Get_Correct_FsFile_Name(const UString &name);
void Correct_FsPath(bool absIsAllowed, UStringVector &parts, bool isDir);
/*
Correct_FsPath() corrects path parts to prepare it for File System operations.
It also corrects empty path parts like "\\\\":
- frontal empty path parts : it removes them or changes them to "_"
- another empty path parts : it removes them
if (absIsAllowed && path is absolute) : it removes empty path parts after start absolute path prefix marker
else
{
if (!keepAndReplaceEmptyPrefixes) : it removes empty path parts
if ( keepAndReplaceEmptyPrefixes) : it changes each empty frontal path part to "_"
}
*/
void Correct_FsPath(bool absIsAllowed, bool keepAndReplaceEmptyPrefixes, UStringVector &parts, bool isDir);
UString MakePathFromParts(const UStringVector &parts);

2
CPP/7zip/UI/Common/OpenArchive.cpp
View File

@@ -857,7 +857,7 @@ HRESULT CArc::GetItem(UInt32 index, CReadArcItem &item) const
if (item.WriteToAltStreamIfColon || needFindAltStream)
{
/* Good handler must support GetRawProps::GetParent for alt streams./
/* Good handler must support GetRawProps::GetParent for alt streams.
So the following code currently is not used */
int colon = FindAltStreamColon_in_Path(item.Path);
if (colon >= 0)

Some files were not shown because too many files have changed in this diff Show More