xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-07 11:14:58 -06:00
Code Issues Packages Projects Releases Wiki Activity

Update to 7-Zip 17.01 Beta from Igor Pavlov

Browse Source 
- Minor speed optimization for LZMA2 (xz and 7z) multi-threading compression.
  7-Zip now uses additional memory buffers for multi-block LZMA2 compression.
  CPU utilization was slightly improved.
- 7-zip now creates multi-block xz archives by default. Block size can be
  specified with -ms[Size]{m|g} switch.
- xz decoder now can unpack random block from multi-block xz archives.  7-Zip
  File Manager now can open nested multi-block xz archives (for example,
  image.iso.xz) without full unpacking of xz archive.
- 7-Zip now can create zip archives from stdin to stdout.
- 7-Zip command line: @listfile now doesn't work after -- switch.  Use
  -i@listfile before -- switch instead.

fixed bugs:
- 7-Zip could add unrequired alternate file streams to WIM archives, for
  commands that contain filename wildcards and -sns switch.
- 7-Zip 17.00 beta crashed for commands that write anti-item to 7z archive.
- 7-Zip 17.00 beta ignored "Use large memory pages" option.
This commit is contained in:
Tino Reichardt
2017-08-28 16:34:04 +02:00
parent 7c1f566312
commit ef790b5209
112 changed files with 4712 additions and 1705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
C/7zCrc.c
View File

@@ -1,5 +1,5 @@
/* 7zCrc.c -- CRC32 init
2017-04-03 : Igor Pavlov : Public domain */
2017-06-06 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -86,8 +86,8 @@ void MY_FAST_CALL CrcGenerateTable()
#ifdef MY_CPU_X86_OR_AMD64
if (!CPU_Is_InOrder())
g_CrcUpdate = CrcUpdateT8;
#endif
g_CrcUpdate = CrcUpdateT8;
#endif
#else
@@ -101,7 +101,7 @@ void MY_FAST_CALL CrcGenerateTable()
g_CrcUpdate = CrcUpdateT4;
#if CRC_NUM_TABLES >= 8
g_CrcUpdateT8 = CrcUpdateT8;
// g_CrcUpdate = CrcUpdateT8;
g_CrcUpdate = CrcUpdateT8;
#endif
}
else if (p[0] != 1 || p[1] != 2)
@@ -118,7 +118,7 @@ void MY_FAST_CALL CrcGenerateTable()
g_CrcUpdate = CrcUpdateT1_BeT4;
#if CRC_NUM_TABLES >= 8
g_CrcUpdateT8 = CrcUpdateT1_BeT8;
// g_CrcUpdate = CrcUpdateT1_BeT8;
g_CrcUpdate = CrcUpdateT1_BeT8;
#endif
}
}

12
C/7zTypes.h
View File

@@ -1,5 +1,5 @@
/* 7zTypes.h -- Basic types
2017-04-03 : Igor Pavlov : Public domain */
2017-07-17 : Igor Pavlov : Public domain */
#ifndef __7Z_TYPES_H
#define __7Z_TYPES_H
@@ -42,13 +42,23 @@ EXTERN_C_BEGIN
typedef int SRes;
#ifdef _WIN32
/* typedef DWORD WRes; */
typedef unsigned WRes;
#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x)
#else
typedef int WRes;
#define MY__FACILITY_WIN32 7
#define MY__FACILITY__WRes MY__FACILITY_WIN32
#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000)))
#endif
#ifndef RINOK
#define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }
#endif

6
C/7zVersion.h
View File

@@ -1,7 +1,7 @@
#define MY_VER_MAJOR 17
#define MY_VER_MINOR 00
#define MY_VER_MINOR 01
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "17.00 ZS v1.3.1 R1"
#define MY_VERSION_NUMBERS "17.01 ZS v1.3.1 R1"
#define MY_VERSION MY_VERSION_NUMBERS
#ifdef MY_CPU_NAME
@@ -10,7 +10,7 @@
#define MY_VERSION_CPU MY_VERSION
#endif
#define MY_DATE "2017-08-20"
#define MY_DATE "2017-08-28"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"

2
C/7zVersionTr.h
View File

@@ -3,7 +3,7 @@
#define MY_VER_BUILD 1
#define MY_VERSION_NUMBERS "1.3.1 R1"
#define MY_VERSION MY_VERSION_NUMBERS
#define MY_DATE "2017-08-20"
#define MY_DATE "2017-08-28"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Tino Reichardt"

4
C/AesOpt.c
View File

@@ -1,12 +1,12 @@
/* AesOpt.c -- Intel's AES
2013-11-12 : Igor Pavlov : Public domain */
2017-06-08 : Igor Pavlov : Public domain */
#include "Precomp.h"
#include "CpuArch.h"
#ifdef MY_CPU_X86_OR_AMD64
#if _MSC_VER >= 1500
#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729)
#define USE_INTEL_AES
#endif
#endif

49
C/Alloc.c
View File

@@ -1,5 +1,5 @@
/* Alloc.c -- Memory allocation functions
2017-04-03 : Igor Pavlov : Public domain */
2017-06-15 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -23,11 +23,11 @@ int g_allocCountBig = 0;
void *MyAlloc(size_t size)
{
if (size == 0)
return 0;
return NULL;
#ifdef _SZ_ALLOC_DEBUG
{
void *p = malloc(size);
fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p);
fprintf(stderr, "\nAlloc %10u bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p);
return p;
}
#else
@@ -38,7 +38,7 @@ void *MyAlloc(size_t size)
void MyFree(void *address)
{
#ifdef _SZ_ALLOC_DEBUG
if (address != 0)
if (address)
fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address);
#endif
free(address);
@@ -49,20 +49,20 @@ void MyFree(void *address)
void *MidAlloc(size_t size)
{
if (size == 0)
return 0;
return NULL;
#ifdef _SZ_ALLOC_DEBUG
fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++);
#endif
return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
}
void MidFree(void *address)
{
#ifdef _SZ_ALLOC_DEBUG
if (address != 0)
if (address)
fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid);
#endif
if (address == 0)
if (!address)
return;
VirtualFree(address, 0, MEM_RELEASE);
}
@@ -79,10 +79,10 @@ typedef SIZE_T (WINAPI *GetLargePageMinimumP)();
void SetLargePageSize()
{
#ifdef _7ZIP_LARGE_PAGES
SIZE_T size = 0;
SIZE_T size;
GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP)
GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum");
if (largePageMinimum == 0)
if (!largePageMinimum)
return;
size = largePageMinimum();
if (size == 0 || (size & (size - 1)) != 0)
@@ -95,31 +95,40 @@ void SetLargePageSize()
void *BigAlloc(size_t size)
{
if (size == 0)
return 0;
return NULL;
#ifdef _SZ_ALLOC_DEBUG
fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++);
fprintf(stderr, "\nAlloc_Big %10u bytes; count = %10d", size, g_allocCountBig++);
#endif
#ifdef _7ZIP_LARGE_PAGES
if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18))
{
void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)),
MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
if (res != 0)
return res;
SIZE_T ps = g_LargePageSize;
if (ps != 0 && ps <= (1 << 30) && size > (ps / 2))
{
size_t size2;
ps--;
size2 = (size + ps) & ~ps;
if (size2 >= size)
{
void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE);
if (res)
return res;
}
}
}
#endif
return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE);
}
void BigFree(void *address)
{
#ifdef _SZ_ALLOC_DEBUG
if (address != 0)
if (address)
fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig);
#endif
if (address == 0)
if (!address)
return;
VirtualFree(address, 0, MEM_RELEASE);
}

22
C/CpuArch.h
View File

@@ -1,5 +1,5 @@
/* CpuArch.h -- CPU specific code
2017-04-03 : Igor Pavlov : Public domain */
2017-06-30 : Igor Pavlov : Public domain */
#ifndef __CPU_ARCH_H
#define __CPU_ARCH_H
@@ -190,8 +190,7 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#ifdef MY_CPU_LE
#if defined(MY_CPU_X86_OR_AMD64) \
|| defined(MY_CPU_ARM64) \
|| defined(__ARM_FEATURE_UNALIGNED) \
|| defined(__AARCH64EL__)
|| defined(__ARM_FEATURE_UNALIGNED)
#define MY_CPU_LE_UNALIGN
#endif
#endif
@@ -237,6 +236,11 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#endif
#ifdef __has_builtin
#define MY__has_builtin(x) __has_builtin(x)
#else
#define MY__has_builtin(x) 0
#endif
#if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300)
@@ -244,15 +248,21 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#include <stdlib.h>
#pragma intrinsic(_byteswap_ushort)
#pragma intrinsic(_byteswap_ulong)
#pragma intrinsic(_byteswap_uint64)
/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */
#define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p))
#define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p))
#define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v)
#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))
#elif defined(MY_CPU_LE_UNALIGN) && ( \
(defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \
|| (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) )
/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */
#define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p))
#define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p))
@@ -277,10 +287,14 @@ MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned mem
#endif
#ifndef GetBe16
#define GetBe16(p) ( (UInt16) ( \
((UInt16)((const Byte *)(p))[0] << 8) | \
((const Byte *)(p))[1] ))
#endif
#ifdef MY_CPU_X86_OR_AMD64

47
C/LzFind.c
View File

@@ -1,5 +1,5 @@
/* LzFind.c -- Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2017-06-10 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -142,6 +142,7 @@ void MatchFinder_Construct(CMatchFinder *p)
p->bufferBase = NULL;
p->directInput = 0;
p->hash = NULL;
p->expectedDataSize = (UInt64)(Int64)-1;
MatchFinder_SetDefaultSettings(p);
for (i = 0; i < 256; i++)
@@ -208,7 +209,11 @@ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize,
hs = (1 << 16) - 1;
else
{
hs = historySize - 1;
hs = historySize;
if (hs > p->expectedDataSize)
hs = (UInt32)p->expectedDataSize;
if (hs != 0)
hs--;
hs |= (hs >> 1);
hs |= (hs >> 2);
hs |= (hs >> 4);
@@ -292,17 +297,33 @@ static void MatchFinder_SetLimits(CMatchFinder *p)
p->posLimit = p->pos + limit;
}
void MatchFinder_Init_2(CMatchFinder *p, int readData)
void MatchFinder_Init_LowHash(CMatchFinder *p)
{
size_t i;
CLzRef *items = p->hash;
size_t numItems = p->fixedHashSize;
for (i = 0; i < numItems; i++)
items[i] = kEmptyHashValue;
}
void MatchFinder_Init_HighHash(CMatchFinder *p)
{
size_t i;
CLzRef *items = p->hash + p->fixedHashSize;
size_t numItems = (size_t)p->hashMask + 1;
for (i = 0; i < numItems; i++)
items[i] = kEmptyHashValue;
}
void MatchFinder_Init_3(CMatchFinder *p, int readData)
{
UInt32 i;
UInt32 *hash = p->hash;
UInt32 num = p->hashSizeSum;
for (i = 0; i < num; i++)
hash[i] = kEmptyHashValue;
p->cyclicBufferPos = 0;
p->buffer = p->bufferBase;
p->pos = p->streamPos = p->cyclicBufferSize;
p->pos =
p->streamPos = p->cyclicBufferSize;
p->result = SZ_OK;
p->streamEndWasReached = 0;
@@ -312,10 +333,14 @@ void MatchFinder_Init_2(CMatchFinder *p, int readData)
MatchFinder_SetLimits(p);
}
void MatchFinder_Init(CMatchFinder *p)
{
MatchFinder_Init_2(p, True);
MatchFinder_Init_HighHash(p);
MatchFinder_Init_LowHash(p);
MatchFinder_Init_3(p, True);
}
static UInt32 MatchFinder_GetSubValue(CMatchFinder *p)
{

8
C/LzFind.h
View File

@@ -1,5 +1,5 @@
/* LzFind.h -- Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2017-06-10 : Igor Pavlov : Public domain */
#ifndef __LZ_FIND_H
#define __LZ_FIND_H
@@ -47,6 +47,8 @@ typedef struct _CMatchFinder
SRes result;
UInt32 crc[256];
size_t numRefs;
UInt64 expectedDataSize;
} CMatchFinder;
#define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)
@@ -103,7 +105,9 @@ typedef struct _IMatchFinder
void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable);
void MatchFinder_Init_2(CMatchFinder *p, int readData);
void MatchFinder_Init_LowHash(CMatchFinder *p);
void MatchFinder_Init_HighHash(CMatchFinder *p);
void MatchFinder_Init_3(CMatchFinder *p, int readData);
void MatchFinder_Init(CMatchFinder *p);
UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);

39
C/LzFindMt.c
View File

@@ -1,5 +1,5 @@
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2017-06-10 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -33,6 +33,8 @@ static void MtSync_GetNextBlock(CMtSync *p)
Event_Set(&p->canStart);
Event_Wait(&p->wasStarted);
// if (mt) MatchFinder_Init_LowHash(mt->MatchFinder);
}
else
{
@@ -155,6 +157,9 @@ static void HashThreadFunc(CMatchFinderMt *mt)
UInt32 numProcessedBlocks = 0;
Event_Wait(&p->canStart);
Event_Set(&p->wasStarted);
MatchFinder_Init_HighHash(mt->MatchFinder);
for (;;)
{
if (p->exit)
@@ -205,7 +210,7 @@ static void HashThreadFunc(CMatchFinderMt *mt)
if (num > kMtHashBlockSize - 2)
num = kMtHashBlockSize - 2;
mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc);
heads[0] += num;
heads[0] = 2 + num;
}
mf->pos += num;
mf->buffer += num;
@@ -496,14 +501,18 @@ SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddB
}
/* Call it after ReleaseStream / SetStream */
void MatchFinderMt_Init(CMatchFinderMt *p)
static void MatchFinderMt_Init(CMatchFinderMt *p)
{
CMatchFinder *mf = p->MatchFinder;
p->btBufPos = p->btBufPosLimit = 0;
p->hashBufPos = p->hashBufPosLimit = 0;
p->btBufPos =
p->btBufPosLimit = 0;
p->hashBufPos =
p->hashBufPosLimit = 0;
/* Init without data reading. We don't want to read data in this thread */
MatchFinder_Init_2(mf, False);
MatchFinder_Init_3(mf, False);
MatchFinder_Init_LowHash(mf);
p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf);
p->btNumAvailBytes = 0;
@@ -684,8 +693,12 @@ static UInt32 MatchFinderMt2_GetMatches(CMatchFinderMt *p, UInt32 *distances)
UInt32 i;
for (i = 0; i < len; i += 2)
{
*distances++ = *btBuf++;
*distances++ = *btBuf++;
UInt32 v0 = btBuf[0];
UInt32 v1 = btBuf[1];
btBuf += 2;
distances[0] = v0;
distances[1] = v1;
distances += 2;
}
}
INCREASE_LZ_POS
@@ -712,8 +725,12 @@ static UInt32 MatchFinderMt_GetMatches(CMatchFinderMt *p, UInt32 *distances)
distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances);
do
{
*distances2++ = *btBuf++;
*distances2++ = *btBuf++;
UInt32 v0 = btBuf[0];
UInt32 v1 = btBuf[1];
btBuf += 2;
distances2[0] = v0;
distances2[1] = v1;
distances2 += 2;
}
while ((len -= 2) != 0);
len = (UInt32)(distances2 - (distances));
@@ -777,7 +794,7 @@ void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable)
{
case 2:
p->GetHeadsFunc = GetHeads2;
p->MixMatchesFunc = (Mf_Mix_Matches)0;
p->MixMatchesFunc = (Mf_Mix_Matches)NULL;
vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip;
vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches;
break;

668
C/Lzma2Enc.c
View File

@@ -1,9 +1,8 @@
/* Lzma2Enc.c -- LZMA2 Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-08-28 : Igor Pavlov : Public domain */
#include "Precomp.h"
/* #include <stdio.h> */
#include <string.h>
/* #define _7ZIP_ST */
@@ -13,7 +12,7 @@
#ifndef _7ZIP_ST
#include "MtCoder.h"
#else
#define NUM_MT_CODER_THREADS_MAX 1
#define MTCODER__THREADS_MAX 1
#endif
#define LZMA2_CONTROL_LZMA (1 << 7)
@@ -35,30 +34,83 @@
#define PRF(x) /* x */
/* ---------- CLimitedSeqInStream ---------- */
typedef struct
{
ISeqInStream vt;
ISeqInStream *realStream;
UInt64 limit;
UInt64 processed;
int finished;
} CLimitedSeqInStream;
static void LimitedSeqInStream_Init(CLimitedSeqInStream *p)
{
p->limit = (UInt64)(Int64)-1;
p->processed = 0;
p->finished = 0;
}
static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size)
{
CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt);
size_t size2 = *size;
SRes res = SZ_OK;
if (p->limit != (UInt64)(Int64)-1)
{
UInt64 rem = p->limit - p->processed;
if (size2 > rem)
size2 = (size_t)rem;
}
if (size2 != 0)
{
res = ISeqInStream_Read(p->realStream, data, &size2);
p->finished = (size2 == 0 ? 1 : 0);
p->processed += size2;
}
*size = size2;
return res;
}
/* ---------- CLzma2EncInt ---------- */
typedef struct
{
CLzmaEncHandle enc;
Byte propsAreSet;
Byte propsByte;
Byte needInitState;
Byte needInitProp;
UInt64 srcPos;
Byte props;
Bool needInitState;
Bool needInitProp;
} CLzma2EncInt;
static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props)
static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props)
{
Byte propsEncoded[LZMA_PROPS_SIZE];
SizeT propsSize = LZMA_PROPS_SIZE;
RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
p->srcPos = 0;
p->props = propsEncoded[0];
p->needInitState = True;
p->needInitProp = True;
if (!p->propsAreSet)
{
SizeT propsSize = LZMA_PROPS_SIZE;
Byte propsEncoded[LZMA_PROPS_SIZE];
RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps));
RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize));
p->propsByte = propsEncoded[0];
p->propsAreSet = True;
}
return SZ_OK;
}
static void Lzma2EncInt_InitBlock(CLzma2EncInt *p)
{
p->srcPos = 0;
p->needInitState = True;
p->needInitProp = True;
}
SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize,
ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
@@ -70,6 +122,9 @@ void LzmaEnc_Finish(CLzmaEncHandle pp);
void LzmaEnc_SaveState(CLzmaEncHandle pp);
void LzmaEnc_RestoreState(CLzmaEncHandle pp);
/*
UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp);
*/
static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
size_t *packSizeRes, ISeqOutStream *outStream)
@@ -154,7 +209,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
outBuf[destPos++] = (Byte)pm;
if (p->needInitProp)
outBuf[destPos++] = p->props;
outBuf[destPos++] = p->propsByte;
p->needInitProp = False;
p->needInitState = False;
@@ -176,14 +231,16 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
void Lzma2EncProps_Init(CLzma2EncProps *p)
{
LzmaEncProps_Init(&p->lzmaProps);
p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO;
p->numBlockThreads_Reduced = -1;
p->numBlockThreads_Max = -1;
p->numTotalThreads = -1;
p->numBlockThreads = -1;
p->blockSize = 0;
}
void Lzma2EncProps_Normalize(CLzma2EncProps *p)
{
int t1, t1n, t2, t3;
UInt64 fileSize;
int t1, t1n, t2, t2r, t3;
{
CLzmaEncProps lzmaProps = p->lzmaProps;
LzmaEncProps_Normalize(&lzmaProps);
@@ -191,11 +248,11 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
}
t1 = p->lzmaProps.numThreads;
t2 = p->numBlockThreads;
t2 = p->numBlockThreads_Max;
t3 = p->numTotalThreads;
if (t2 > NUM_MT_CODER_THREADS_MAX)
t2 = NUM_MT_CODER_THREADS_MAX;
if (t2 > MTCODER__THREADS_MAX)
t2 = MTCODER__THREADS_MAX;
if (t3 <= 0)
{
@@ -211,8 +268,8 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
t1 = 1;
t2 = t3;
}
if (t2 > NUM_MT_CODER_THREADS_MAX)
t2 = NUM_MT_CODER_THREADS_MAX;
if (t2 > MTCODER__THREADS_MAX)
t2 = MTCODER__THREADS_MAX;
}
else if (t1 <= 0)
{
@@ -225,39 +282,64 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p)
p->lzmaProps.numThreads = t1;
t2r = t2;
fileSize = p->lzmaProps.reduceSize;
if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
&& p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
&& (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1))
p->lzmaProps.reduceSize = p->blockSize;
LzmaEncProps_Normalize(&p->lzmaProps);
p->lzmaProps.reduceSize = fileSize;
t1 = p->lzmaProps.numThreads;
if (p->blockSize == 0)
if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID)
{
UInt32 dictSize = p->lzmaProps.dictSize;
UInt64 blockSize = (UInt64)dictSize << 2;
const UInt32 kMinSize = (UInt32)1 << 20;
const UInt32 kMaxSize = (UInt32)1 << 28;
if (blockSize < kMinSize) blockSize = kMinSize;
if (blockSize > kMaxSize) blockSize = kMaxSize;
if (blockSize < dictSize) blockSize = dictSize;
p->blockSize = (size_t)blockSize;
t2r = t2 = 1;
t3 = t1;
}
if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1)
else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1)
{
UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1;
if (temp > p->lzmaProps.reduceSize)
/* if there is no block multi-threading, we use SOLID block */
p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID;
}
else
{
if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO)
{
UInt64 numBlocks = temp / p->blockSize;
const UInt32 kMinSize = (UInt32)1 << 20;
const UInt32 kMaxSize = (UInt32)1 << 28;
const UInt32 dictSize = p->lzmaProps.dictSize;
UInt64 blockSize = (UInt64)dictSize << 2;
if (blockSize < kMinSize) blockSize = kMinSize;
if (blockSize > kMaxSize) blockSize = kMaxSize;
if (blockSize < dictSize) blockSize = dictSize;
blockSize += (kMinSize - 1);
blockSize &= ~(UInt64)(kMinSize - 1);
p->blockSize = blockSize;
}
if (t2 > 1 && fileSize != (UInt64)(Int64)-1)
{
UInt64 numBlocks = fileSize / p->blockSize;
if (numBlocks * p->blockSize != fileSize)
numBlocks++;
if (numBlocks < (unsigned)t2)
{
t2 = (unsigned)numBlocks;
if (t2 == 0)
t2 = 1;
t3 = t1 * t2;
t2r = (unsigned)numBlocks;
if (t2r == 0)
t2r = 1;
t3 = t1 * t2r;
}
}
}
p->numBlockThreads = t2;
p->numBlockThreads_Max = t2;
p->numBlockThreads_Reduced = t2r;
p->numTotalThreads = t3;
}
@@ -274,134 +356,30 @@ typedef struct
{
Byte propEncoded;
CLzma2EncProps props;
UInt64 expectedDataSize;
Byte *outBuf;
Byte *tempBufLzma;
ISzAllocPtr alloc;
ISzAllocPtr allocBig;
CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX];
CLzma2EncInt coders[MTCODER__THREADS_MAX];
#ifndef _7ZIP_ST
ISeqOutStream *outStream;
Byte *outBuf;
size_t outBufSize;
size_t outBufsDataSizes[MTCODER__BLOCKS_MAX];
Bool mtCoder_WasConstructed;
CMtCoder mtCoder;
Byte *outBufs[MTCODER__BLOCKS_MAX];
#endif
} CLzma2Enc;
/* ---------- Lzma2EncThread ---------- */
static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
UInt64 packTotal = 0;
SRes res = SZ_OK;
if (!mainEncoder->outBuf)
{
mainEncoder->outBuf = (Byte *)ISzAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
if (!mainEncoder->outBuf)
return SZ_ERROR_MEM;
}
RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE,
mainEncoder->alloc, mainEncoder->allocBig));
for (;;)
{
size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream);
if (res != SZ_OK)
break;
packTotal += packSize;
res = Progress(progress, p->srcPos, packTotal);
if (res != SZ_OK)
break;
if (packSize == 0)
break;
}
LzmaEnc_Finish(p->enc);
if (res == SZ_OK)
{
Byte b = 0;
if (ISeqOutStream_Write(outStream, &b, 1) != 1)
return SZ_ERROR_WRITE;
}
return res;
}
#ifndef _7ZIP_ST
typedef struct
{
IMtCoderCallback funcTable;
CLzma2Enc *lzma2Enc;
} CMtCallbackImp;
static SRes MtCallbackImp_Code(const IMtCoderCallback *pp, unsigned index, Byte *dest, size_t *destSize,
const Byte *src, size_t srcSize, int finished)
{
CMtCallbackImp *imp = CONTAINER_FROM_VTBL(pp, CMtCallbackImp, funcTable);
CLzma2Enc *mainEncoder = imp->lzma2Enc;
CLzma2EncInt *p = &mainEncoder->coders[index];
SRes res = SZ_OK;
{
size_t destLim = *destSize;
*destSize = 0;
if (srcSize != 0)
{
RINOK(Lzma2EncInt_Init(p, &mainEncoder->props));
RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE,
mainEncoder->alloc, mainEncoder->allocBig));
while (p->srcPos < srcSize)
{
size_t packSize = destLim - *destSize;
res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL);
if (res != SZ_OK)
break;
*destSize += packSize;
if (packSize == 0)
{
res = SZ_ERROR_FAIL;
break;
}
if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK)
{
res = SZ_ERROR_PROGRESS;
break;
}
}
LzmaEnc_Finish(p->enc);
if (res != SZ_OK)
return res;
}
if (finished)
{
if (*destSize == destLim)
return SZ_ERROR_OUTPUT_EOF;
dest[(*destSize)++] = 0;
}
}
return res;
}
#endif
/* ---------- Lzma2Enc ---------- */
CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)
{
@@ -410,44 +388,78 @@ CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig)
return NULL;
Lzma2EncProps_Init(&p->props);
Lzma2EncProps_Normalize(&p->props);
p->outBuf = 0;
p->expectedDataSize = (UInt64)(Int64)-1;
p->tempBufLzma = NULL;
p->alloc = alloc;
p->allocBig = allocBig;
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->coders[i].enc = 0;
for (i = 0; i < MTCODER__THREADS_MAX; i++)
p->coders[i].enc = NULL;
}
#ifndef _7ZIP_ST
MtCoder_Construct(&p->mtCoder);
p->mtCoder_WasConstructed = False;
{
unsigned i;
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
p->outBufs[i] = NULL;
p->outBufSize = 0;
}
#endif
return p;
}
#ifndef _7ZIP_ST
static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p)
{
unsigned i;
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
if (p->outBufs[i])
{
ISzAlloc_Free(p->alloc, p->outBufs[i]);
p->outBufs[i] = NULL;
}
p->outBufSize = 0;
}
#endif
void Lzma2Enc_Destroy(CLzma2EncHandle pp)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CLzma2EncInt *t = &p->coders[i];
if (t->enc)
{
LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig);
t->enc = 0;
t->enc = NULL;
}
}
#ifndef _7ZIP_ST
MtCoder_Destruct(&p->mtCoder);
if (p->mtCoder_WasConstructed)
{
MtCoder_Destruct(&p->mtCoder);
p->mtCoder_WasConstructed = False;
}
Lzma2Enc_FreeOutBufs(p);
#endif
ISzAlloc_Free(p->alloc, p->outBuf);
ISzAlloc_Free(p->alloc, p->tempBufLzma);
p->tempBufLzma = NULL;
ISzAlloc_Free(p->alloc, pp);
}
SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
@@ -460,6 +472,14 @@ SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props)
return SZ_OK;
}
void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
p->expectedDataSize = expectedDataSiize;
}
Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
@@ -471,50 +491,310 @@ Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp)
return (Byte)i;
}
SRes Lzma2Enc_Encode(CLzma2EncHandle pp,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
int i;
for (i = 0; i < p->props.numBlockThreads; i++)
static SRes Lzma2Enc_EncodeMt1(
CLzma2Enc *me,
CLzma2EncInt *p,
ISeqOutStream *outStream,
Byte *outBuf, size_t *outBufSize,
ISeqInStream *inStream,
const Byte *inData, size_t inDataSize,
int finished,
ICompressProgress *progress)
{
UInt64 unpackTotal = 0;
UInt64 packTotal = 0;
size_t outLim = 0;
CLimitedSeqInStream limitedInStream;
if (outBuf)
{
CLzma2EncInt *t = &p->coders[(unsigned)i];
if (!t->enc)
outLim = *outBufSize;
*outBufSize = 0;
}
if (!p->enc)
{
p->propsAreSet = False;
p->enc = LzmaEnc_Create(me->alloc);
if (!p->enc)
return SZ_ERROR_MEM;
}
limitedInStream.realStream = inStream;
if (inStream)
{
limitedInStream.vt.Read = LimitedSeqInStream_Read;
}
if (!outBuf)
{
// outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma
if (!me->tempBufLzma)
{
t->enc = LzmaEnc_Create(p->alloc);
if (!t->enc)
me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX);
if (!me->tempBufLzma)
return SZ_ERROR_MEM;
}
}
#ifndef _7ZIP_ST
if (p->props.numBlockThreads > 1)
{
CMtCallbackImp mtCallback;
RINOK(Lzma2EncInt_InitStream(p, &me->props));
mtCallback.funcTable.Code = MtCallbackImp_Code;
mtCallback.lzma2Enc = p;
for (;;)
{
SRes res = SZ_OK;
size_t inSizeCur = 0;
Lzma2EncInt_InitBlock(p);
LimitedSeqInStream_Init(&limitedInStream);
limitedInStream.limit = me->props.blockSize;
if (inStream)
{
UInt64 expected = (UInt64)(Int64)-1;
// inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize
if (me->expectedDataSize != (UInt64)(Int64)-1
&& me->expectedDataSize >= unpackTotal)
expected = me->expectedDataSize - unpackTotal;
if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
&& expected > me->props.blockSize)
expected = (size_t)me->props.blockSize;
LzmaEnc_SetDataSize(p->enc, expected);
RINOK(LzmaEnc_PrepareForLzma2(p->enc,
&limitedInStream.vt,
LZMA2_KEEP_WINDOW_SIZE,
me->alloc,
me->allocBig));
}
else
{
inSizeCur = inDataSize - (size_t)unpackTotal;
if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
&& inSizeCur > me->props.blockSize)
inSizeCur = (size_t)me->props.blockSize;
// LzmaEnc_SetDataSize(p->enc, inSizeCur);
RINOK(LzmaEnc_MemPrepare(p->enc,
inData + (size_t)unpackTotal, inSizeCur,
LZMA2_KEEP_WINDOW_SIZE,
me->alloc,
me->allocBig));
}
for (;;)
{
size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX;
if (outBuf)
packSize = outLim - (size_t)packTotal;
res = Lzma2EncInt_EncodeSubblock(p,
outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize,
outBuf ? NULL : outStream);
if (res != SZ_OK)
break;
packTotal += packSize;
if (outBuf)
*outBufSize = (size_t)packTotal;
res = Progress(progress, unpackTotal + p->srcPos, packTotal);
if (res != SZ_OK)
break;
/*
if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0)
break;
*/
if (packSize == 0)
break;
}
LzmaEnc_Finish(p->enc);
unpackTotal += p->srcPos;
RINOK(res);
if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur))
return SZ_ERROR_FAIL;
if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize))
{
if (finished)
{
if (outBuf)
{
size_t destPos = *outBufSize;
if (destPos >= outLim)
return SZ_ERROR_OUTPUT_EOF;
outBuf[destPos] = 0;
*outBufSize = destPos + 1;
}
else
{
Byte b = 0;
if (ISeqOutStream_Write(outStream, &b, 1) != 1)
return SZ_ERROR_WRITE;
}
}
return SZ_OK;
}
}
}
#ifndef _7ZIP_ST
static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex,
const Byte *src, size_t srcSize, int finished)
{
CLzma2Enc *me = (CLzma2Enc *)pp;
size_t destSize = me->outBufSize;
SRes res;
CMtProgressThunk progressThunk;
Byte *dest = me->outBufs[outBufIndex];
me->outBufsDataSizes[outBufIndex] = 0;
if (!dest)
{
dest = ISzAlloc_Alloc(me->alloc, me->outBufSize);
if (!dest)
return SZ_ERROR_MEM;
me->outBufs[outBufIndex] = dest;
}
MtProgressThunk_CreateVTable(&progressThunk);
progressThunk.mtProgress = &me->mtCoder.mtProgress;
progressThunk.index = coderIndex;
res = Lzma2Enc_EncodeMt1(me,
&me->coders[coderIndex],
NULL, dest, &destSize,
NULL, src, srcSize,
finished,
&progressThunk.vt);
me->outBufsDataSizes[outBufIndex] = destSize;
return res;
}
static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex)
{
CLzma2Enc *me = (CLzma2Enc *)pp;
size_t size = me->outBufsDataSizes[outBufIndex];
const Byte *data = me->outBufs[outBufIndex];
if (me->outStream)
return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE;
if (size > me->outBufSize)
return SZ_ERROR_OUTPUT_EOF;
memcpy(me->outBuf, data, size);
me->outBufSize -= size;
me->outBuf += size;
return SZ_OK;
}
#endif
SRes Lzma2Enc_Encode2(CLzma2EncHandle pp,
ISeqOutStream *outStream,
Byte *outBuf, size_t *outBufSize,
ISeqInStream *inStream,
const Byte *inData, size_t inDataSize,
ICompressProgress *progress)
{
CLzma2Enc *p = (CLzma2Enc *)pp;
if (inStream && inData)
return E_INVALIDARG;
if (outStream && outBuf)
return E_INVALIDARG;
{
unsigned i;
for (i = 0; i < MTCODER__THREADS_MAX; i++)
p->coders[i].propsAreSet = False;
}
#ifndef _7ZIP_ST
if (p->props.numBlockThreads_Reduced > 1)
{
IMtCoderCallback2 vt;
if (!p->mtCoder_WasConstructed)
{
p->mtCoder_WasConstructed = True;
MtCoder_Construct(&p->mtCoder);
}
vt.Code = Lzma2Enc_MtCallback_Code;
vt.Write = Lzma2Enc_MtCallback_Write;
p->outStream = outStream;
p->outBuf = NULL;
p->outBufSize = 0;
if (!outStream)
{
p->outBuf = outBuf;
p->outBufSize = *outBufSize;
*outBufSize = 0;
}
p->mtCoder.allocBig = p->allocBig;
p->mtCoder.progress = progress;
p->mtCoder.inStream = inStream;
p->mtCoder.outStream = outStream;
p->mtCoder.alloc = p->alloc;
p->mtCoder.mtCallback = &mtCallback.funcTable;
p->mtCoder.inData = inData;
p->mtCoder.inDataSize = inDataSize;
p->mtCoder.mtCallback = &vt;
p->mtCoder.mtCallbackObject = p;
p->mtCoder.blockSize = (size_t)p->props.blockSize;
if (p->mtCoder.blockSize != p->props.blockSize)
return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */
p->mtCoder.blockSize = p->props.blockSize;
p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16;
if (p->mtCoder.destBlockSize < p->props.blockSize)
{
p->mtCoder.destBlockSize = (size_t)0 - 1;
if (p->mtCoder.destBlockSize < p->props.blockSize)
return SZ_ERROR_FAIL;
size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16;
if (destBlockSize < p->mtCoder.blockSize)
return SZ_ERROR_PARAM;
if (p->outBufSize != destBlockSize)
Lzma2Enc_FreeOutBufs(p);
p->outBufSize = destBlockSize;
}
p->mtCoder.numThreads = p->props.numBlockThreads;
p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max;
p->mtCoder.expectedDataSize = p->expectedDataSize;
return MtCoder_Code(&p->mtCoder);
{
SRes res = MtCoder_Code(&p->mtCoder);
if (!outStream)
*outBufSize = p->outBuf - outBuf;
return res;
}
}
#endif
return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress);
return Lzma2Enc_EncodeMt1(p,
&p->coders[0],
outStream, outBuf, outBufSize,
inStream, inData, inDataSize,
True, /* finished */
progress);
}

43
C/Lzma2Enc.h
View File

@@ -1,5 +1,5 @@
/* Lzma2Enc.h -- LZMA2 Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#ifndef __LZMA2_ENC_H
#define __LZMA2_ENC_H
@@ -8,11 +8,15 @@
EXTERN_C_BEGIN
#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0
#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1)
typedef struct
{
CLzmaEncProps lzmaProps;
size_t blockSize;
int numBlockThreads;
UInt64 blockSize;
int numBlockThreads_Reduced;
int numBlockThreads_Max;
int numTotalThreads;
} CLzma2EncProps;
@@ -22,13 +26,14 @@ void Lzma2EncProps_Normalize(CLzma2EncProps *p);
/* ---------- CLzmaEnc2Handle Interface ---------- */
/* Lzma2Enc_* functions can return the following exit codes:
Returns:
SRes:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater in props
SZ_ERROR_WRITE - Write callback error
SZ_ERROR_WRITE - ISeqOutStream write callback error
SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
SZ_ERROR_PROGRESS - some break from progress callback
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
*/
typedef void * CLzma2EncHandle;
@@ -36,26 +41,14 @@ typedef void * CLzma2EncHandle;
CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig);
void Lzma2Enc_Destroy(CLzma2EncHandle p);
SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props);
void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize);
Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p);
SRes Lzma2Enc_Encode(CLzma2EncHandle p,
ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
/* ---------- One Call Interface ---------- */
/* Lzma2Encode
Return code:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater
SZ_ERROR_OUTPUT_EOF - output buffer overflow
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
/*
SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark,
ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
*/
SRes Lzma2Enc_Encode2(CLzma2EncHandle p,
ISeqOutStream *outStream,
Byte *outBuf, size_t *outBufSize,
ISeqInStream *inStream,
const Byte *inData, size_t inDataSize,
ICompressProgress *progress);
EXTERN_C_END

47
C/LzmaEnc.c
View File

@@ -1,5 +1,5 @@
/* LzmaEnc.c -- LZMA Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-06-22 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -23,8 +23,8 @@
static unsigned g_STAT_OFFSET = 0;
#endif
#define kMaxHistorySize ((UInt32)3 << 29)
/* #define kMaxHistorySize ((UInt32)7 << 29) */
#define kLzmaMaxHistorySize ((UInt32)3 << 29)
/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */
#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1)
@@ -62,14 +62,15 @@ void LzmaEncProps_Normalize(CLzmaEncProps *p)
if (level < 0) level = 5;
p->level = level;
if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26)));
if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26)));
if (p->dictSize > p->reduceSize)
{
unsigned i;
UInt32 reduceSize = (UInt32)p->reduceSize;
for (i = 11; i <= 30; i++)
{
if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; }
if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; }
if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; }
if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; }
}
}
@@ -445,7 +446,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
|| props.lp > LZMA_LP_MAX
|| props.pb > LZMA_PB_MAX
|| props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress)
|| props.dictSize > kMaxHistorySize)
|| props.dictSize > kLzmaMaxHistorySize)
return SZ_ERROR_PARAM;
p->dictSize = props.dictSize;
@@ -492,6 +493,15 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2)
return SZ_OK;
}
void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize)
{
CLzmaEnc *p = (CLzmaEnc *)pp;
p->matchFinderBase.expectedDataSize = expectedDataSiize;
}
static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5};
static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10};
static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11};
@@ -1996,6 +2006,8 @@ static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc,
{
RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig));
p->matchFinderObj = &p->matchFinderMt;
p->matchFinderBase.bigHash = (Byte)(
(p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0);
MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder);
}
else
@@ -2135,6 +2147,7 @@ SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
LzmaEnc_SetInputBuf(p, src, srcLen);
p->needInit = 1;
LzmaEnc_SetDataSize(pp, srcLen);
return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig);
}
@@ -2152,15 +2165,15 @@ void LzmaEnc_Finish(CLzmaEncHandle pp)
typedef struct
{
ISeqOutStream funcTable;
ISeqOutStream vt;
Byte *data;
SizeT rem;
Bool overflow;
} CSeqOutStreamBuf;
} CLzmaEnc_SeqOutStreamBuf;
static size_t MyWrite(const ISeqOutStream *pp, const void *data, size_t size)
static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size)
{
CSeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CSeqOutStreamBuf, funcTable);
CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt);
if (p->rem < size)
{
size = p->rem;
@@ -2193,9 +2206,9 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
CLzmaEnc *p = (CLzmaEnc *)pp;
UInt64 nowPos64;
SRes res;
CSeqOutStreamBuf outStream;
CLzmaEnc_SeqOutStreamBuf outStream;
outStream.funcTable.Write = MyWrite;
outStream.vt.Write = SeqOutStreamBuf_Write;
outStream.data = dest;
outStream.rem = *destLen;
outStream.overflow = False;
@@ -2209,7 +2222,7 @@ SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
LzmaEnc_InitPrices(p);
nowPos64 = p->nowPos64;
RangeEnc_Init(&p->rc);
p->rc.outStream = &outStream.funcTable;
p->rc.outStream = &outStream.vt;
res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize);
@@ -2308,15 +2321,15 @@ SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte
SRes res;
CLzmaEnc *p = (CLzmaEnc *)pp;
CSeqOutStreamBuf outStream;
CLzmaEnc_SeqOutStreamBuf outStream;
outStream.funcTable.Write = MyWrite;
outStream.vt.Write = SeqOutStreamBuf_Write;
outStream.data = dest;
outStream.rem = *destLen;
outStream.overflow = False;
p->writeEndMark = writeEndMark;
p->rc.outStream = &outStream.funcTable;
p->rc.outStream = &outStream.vt;
res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig);

37
C/LzmaEnc.h
View File

@@ -1,5 +1,5 @@
/* LzmaEnc.h -- LZMA Encoder
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#ifndef __LZMA_ENC_H
#define __LZMA_ENC_H
@@ -12,12 +12,10 @@ EXTERN_C_BEGIN
typedef struct _CLzmaEncProps
{
int level; /* 0 <= level <= 9 */
int level; /* 0 <= level <= 9 */
UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version
(1 << 12) <= dictSize <= (1 << 30) for 64-bit version
default = (1 << 24) */
UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF.
Encoder uses this value to reduce dictionary size */
(1 << 12) <= dictSize <= (3 << 29) for 64-bit version
default = (1 << 24) */
int lc; /* 0 <= lc <= 8, default = 3 */
int lp; /* 0 <= lp <= 4, default = 0 */
int pb; /* 0 <= pb <= 4, default = 2 */
@@ -25,9 +23,12 @@ typedef struct _CLzmaEncProps
int fb; /* 5 <= fb <= 273, default = 32 */
int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */
int numHashBytes; /* 2, 3 or 4, default = 4 */
UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */
UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */
unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */
int numThreads; /* 1 or 2, default = 2 */
UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1.
Encoder uses this value to reduce dictionary size */
} CLzmaEncProps;
void LzmaEncProps_Init(CLzmaEncProps *p);
@@ -37,38 +38,34 @@ UInt32 LzmaEncProps_GetDictSize(const CLzmaEncProps *props2);
/* ---------- CLzmaEncHandle Interface ---------- */
/* LzmaEnc_* functions can return the following exit codes:
Returns:
/* LzmaEnc* functions can return the following exit codes:
SRes:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater in props
SZ_ERROR_WRITE - Write callback error.
SZ_ERROR_WRITE - ISeqOutStream write callback error
SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output
SZ_ERROR_PROGRESS - some break from progress callback
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
SZ_ERROR_THREAD - error in multithreading functions (only for Mt version)
*/
typedef void * CLzmaEncHandle;
CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc);
void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props);
void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize);
SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size);
unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p);
SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream,
ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig);
/* ---------- One Call Interface ---------- */
/* LzmaEncode
Return code:
SZ_OK - OK
SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_PARAM - Incorrect paramater
SZ_ERROR_OUTPUT_EOF - output buffer overflow
SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version)
*/
/* ---------- One Call Interface ---------- */
SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen,
const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark,

745
C/MtCoder.c
View File

@@ -1,95 +1,72 @@
/* MtCoder.c -- Multi-thread Coder
2017-04-03 : Igor Pavlov : Public domain */
2017-07-17 : Igor Pavlov : Public domain */
#include "Precomp.h"
#include "MtCoder.h"
void LoopThread_Construct(CLoopThread *p)
{
Thread_Construct(&p->thread);
Event_Construct(&p->startEvent);
Event_Construct(&p->finishedEvent);
}
void LoopThread_Close(CLoopThread *p)
{
Thread_Close(&p->thread);
Event_Close(&p->startEvent);
Event_Close(&p->finishedEvent);
}
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp)
{
CLoopThread *p = (CLoopThread *)pp;
for (;;)
{
if (Event_Wait(&p->startEvent) != 0)
return SZ_ERROR_THREAD;
if (p->stop)
return 0;
p->res = p->func(p->param);
if (Event_Set(&p->finishedEvent) != 0)
return SZ_ERROR_THREAD;
}
}
WRes LoopThread_Create(CLoopThread *p)
{
p->stop = 0;
RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent));
RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent));
return Thread_Create(&p->thread, LoopThreadFunc, p);
}
WRes LoopThread_StopAndWait(CLoopThread *p)
{
p->stop = 1;
if (Event_Set(&p->startEvent) != 0)
return SZ_ERROR_THREAD;
return Thread_Wait(&p->thread);
}
WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); }
WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); }
static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize)
{
return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK;
}
static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress)
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->inSizes[i] = p->outSizes[i] = 0;
p->totalInSize = p->totalOutSize = 0;
p->progress = progress;
p->res = SZ_OK;
p->totalInSize = 0;
p->totalOutSize = 0;
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CMtProgressSizes *pair = &p->sizes[i];
pair->inSize = 0;
pair->outSize = 0;
}
}
static void MtProgress_Reinit(CMtProgress *p, unsigned index)
{
p->inSizes[index] = 0;
p->outSizes[index] = 0;
CMtProgressSizes *pair = &p->sizes[index];
pair->inSize = 0;
pair->outSize = 0;
}
#define UPDATE_PROGRESS(size, prev, total) \
if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; }
SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize)
{
SRes res;
CMtProgressSizes *pair;
CriticalSection_Enter(&p->cs);
pair = &p->sizes[index];
UPDATE_PROGRESS(inSize, pair->inSize, p->totalInSize)
UPDATE_PROGRESS(outSize, pair->outSize, p->totalOutSize)
if (p->res == SZ_OK && p->progress)
{
if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK)
p->res = SZ_ERROR_PROGRESS;
}
res = p->res;
CriticalSection_Leave(&p->cs);
return res;
}
static SRes MtProgress_GetError(CMtProgress *p)
{
SRes res;
CriticalSection_Enter(&p->cs);
UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize)
UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize)
if (p->res == SZ_OK)
p->res = Progress(p->progress, p->totalInSize, p->totalOutSize);
res = p->res;
CriticalSection_Leave(&p->cs);
return res;
}
static void MtProgress_SetError(CMtProgress *p, SRes res)
{
CriticalSection_Enter(&p->cs);
@@ -98,71 +75,72 @@ static void MtProgress_SetError(CMtProgress *p, SRes res)
CriticalSection_Leave(&p->cs);
}
static void MtCoder_SetError(CMtCoder* p, SRes res)
static SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize)
{
CriticalSection_Enter(&p->cs);
if (p->res == SZ_OK)
p->res = res;
CriticalSection_Leave(&p->cs);
CMtProgressThunk *p = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt);
return MtProgress_Set(p->mtProgress, p->index, inSize, outSize);
}
/* ---------- MtThread ---------- */
void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder)
void MtProgressThunk_CreateVTable(CMtProgressThunk *p)
{
p->mtCoder = mtCoder;
p->outBuf = 0;
p->inBuf = 0;
Event_Construct(&p->canRead);
Event_Construct(&p->canWrite);
LoopThread_Construct(&p->thread);
p->vt.Progress = MtProgressThunk_Progress;
}
#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; }
static void CMtThread_CloseEvents(CMtThread *p)
static WRes ArEvent_OptCreate_And_Reset(CEvent *p)
{
Event_Close(&p->canRead);
Event_Close(&p->canWrite);
if (Event_IsCreated(p))
return Event_Reset(p);
return AutoResetEvent_CreateNotSignaled(p);
}
static void CMtThread_Destruct(CMtThread *p)
{
CMtThread_CloseEvents(p);
if (Thread_WasCreated(&p->thread.thread))
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp);
static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t)
{
WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent);
if (wres == 0)
{
LoopThread_StopAndWait(&p->thread);
LoopThread_Close(&p->thread);
t->stop = False;
if (!Thread_WasCreated(&t->thread))
wres = Thread_Create(&t->thread, ThreadFunc, t);
if (wres == 0)
wres = Event_Set(&t->startEvent);
}
if (wres == 0)
return SZ_OK;
return MY_SRes_HRESULT_FROM_WRes(wres);
}
static void MtCoderThread_Destruct(CMtCoderThread *t)
{
if (Thread_WasCreated(&t->thread))
{
t->stop = 1;
Event_Set(&t->startEvent);
Thread_Wait(&t->thread);
Thread_Close(&t->thread);
}
if (p->mtCoder->alloc)
ISzAlloc_Free(p->mtCoder->alloc, p->outBuf);
p->outBuf = 0;
Event_Close(&t->startEvent);
if (p->mtCoder->alloc)
ISzAlloc_Free(p->mtCoder->alloc, p->inBuf);
p->inBuf = 0;
if (t->inBuf)
{
ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf);
t->inBuf = NULL;
}
}
#define MY_BUF_ALLOC(buf, size, newSize) \
if (buf == 0 || size != newSize) \
{ ISzAlloc_Free(p->mtCoder->alloc, buf); \
size = newSize; buf = (Byte *)ISzAlloc_Alloc(p->mtCoder->alloc, size); \
if (buf == 0) return SZ_ERROR_MEM; }
static SRes CMtThread_Prepare(CMtThread *p)
{
MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize)
MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize)
p->stopReading = False;
p->stopWriting = False;
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead));
RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite));
return SZ_OK;
}
static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
{
@@ -170,158 +148,509 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
*processedSize = 0;
while (size != 0)
{
size_t curSize = size;
SRes res = ISeqInStream_Read(stream, data, &curSize);
*processedSize += curSize;
data += curSize;
size -= curSize;
size_t cur = size;
SRes res = ISeqInStream_Read(stream, data, &cur);
*processedSize += cur;
data += cur;
size -= cur;
RINOK(res);
if (curSize == 0)
if (cur == 0)
return SZ_OK;
}
return SZ_OK;
}
#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1]
static SRes MtThread_Process(CMtThread *p, Bool *stop)
/*
ThreadFunc2() returns:
SZ_OK - in all normal cases (even for stream error or memory allocation error)
SZ_ERROR_THREAD - in case of failure in system synch function
*/
static SRes ThreadFunc2(CMtCoderThread *t)
{
CMtThread *next;
*stop = True;
if (Event_Wait(&p->canRead) != 0)
return SZ_ERROR_THREAD;
next = GET_NEXT_THREAD(p);
if (p->stopReading)
{
next->stopReading = True;
return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD;
}
CMtCoder *mtc = t->mtCoder;
{
size_t size = p->mtCoder->blockSize;
size_t destSize = p->outBufSize;
RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size));
next->stopReading = *stop = (size != p->mtCoder->blockSize);
if (Event_Set(&next->canRead) != 0)
return SZ_ERROR_THREAD;
RINOK(IMtCoderCallback_Code(p->mtCoder->mtCallback, p->index,
p->outBuf, &destSize, p->inBuf, size, *stop));
MtProgress_Reinit(&p->mtCoder->mtProgress, p->index);
if (Event_Wait(&p->canWrite) != 0)
return SZ_ERROR_THREAD;
if (p->stopWriting)
return SZ_ERROR_FAIL;
if (ISeqOutStream_Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize)
return SZ_ERROR_WRITE;
return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD;
}
}
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
{
CMtThread *p = (CMtThread *)pp;
for (;;)
{
Bool stop;
CMtThread *next = GET_NEXT_THREAD(p);
SRes res = MtThread_Process(p, &stop);
if (res != SZ_OK)
unsigned bi;
SRes res;
SRes res2;
Bool finished;
unsigned bufIndex;
size_t size;
const Byte *inData;
UInt64 readProcessed = 0;
RINOK_THREAD(Event_Wait(&mtc->readEvent))
/* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */
if (mtc->stopReading)
{
MtCoder_SetError(p->mtCoder, res);
MtProgress_SetError(&p->mtCoder->mtProgress, res);
next->stopReading = True;
next->stopWriting = True;
Event_Set(&next->canRead);
Event_Set(&next->canWrite);
return res;
return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD;
}
if (stop)
res = MtProgress_GetError(&mtc->mtProgress);
size = 0;
inData = NULL;
finished = True;
if (res == SZ_OK)
{
size = mtc->blockSize;
if (mtc->inStream)
{
if (!t->inBuf)
{
t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize);
if (!t->inBuf)
res = SZ_ERROR_MEM;
}
if (res == SZ_OK)
{
res = FullRead(mtc->inStream, t->inBuf, &size);
readProcessed = mtc->readProcessed + size;
mtc->readProcessed = readProcessed;
}
if (res != SZ_OK)
{
mtc->readRes = res;
/* after reading error - we can stop encoding of previous blocks */
MtProgress_SetError(&mtc->mtProgress, res);
}
else
finished = (size != mtc->blockSize);
}
else
{
size_t rem;
readProcessed = mtc->readProcessed;
rem = mtc->inDataSize - (size_t)readProcessed;
if (size > rem)
size = rem;
inData = mtc->inData + (size_t)readProcessed;
readProcessed += size;
mtc->readProcessed = readProcessed;
finished = (mtc->inDataSize == (size_t)readProcessed);
}
}
/* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */
res2 = SZ_OK;
if (Semaphore_Wait(&mtc->blocksSemaphore) != 0)
{
res2 = SZ_ERROR_THREAD;
if (res == SZ_OK)
{
res = res2;
// MtProgress_SetError(&mtc->mtProgress, res);
}
}
bi = mtc->blockIndex;
if (++mtc->blockIndex >= mtc->numBlocksMax)
mtc->blockIndex = 0;
bufIndex = (unsigned)(int)-1;
if (res == SZ_OK)
res = MtProgress_GetError(&mtc->mtProgress);
if (res != SZ_OK)
finished = True;
if (!finished)
{
if (mtc->numStartedThreads < mtc->numStartedThreadsLimit
&& mtc->expectedDataSize != readProcessed)
{
res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]);
if (res == SZ_OK)
mtc->numStartedThreads++;
else
{
MtProgress_SetError(&mtc->mtProgress, res);
finished = True;
}
}
}
if (finished)
mtc->stopReading = True;
RINOK_THREAD(Event_Set(&mtc->readEvent))
if (res2 != SZ_OK)
return res2;
if (res == SZ_OK)
{
CriticalSection_Enter(&mtc->cs);
bufIndex = mtc->freeBlockHead;
mtc->freeBlockHead = mtc->freeBlockList[bufIndex];
CriticalSection_Leave(&mtc->cs);
res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex,
mtc->inStream ? t->inBuf : inData, size, finished);
MtProgress_Reinit(&mtc->mtProgress, t->index);
if (res != SZ_OK)
MtProgress_SetError(&mtc->mtProgress, res);
}
{
CMtCoderBlock *block = &mtc->blocks[bi];
block->res = res;
block->bufIndex = bufIndex;
block->finished = finished;
}
#ifdef MTCODER__USE_WRITE_THREAD
RINOK_THREAD(Event_Set(&mtc->writeEvents[bi]))
#else
{
unsigned wi;
{
CriticalSection_Enter(&mtc->cs);
wi = mtc->writeIndex;
if (wi == bi)
mtc->writeIndex = (unsigned)(int)-1;
else
mtc->ReadyBlocks[bi] = True;
CriticalSection_Leave(&mtc->cs);
}
if (wi != bi)
{
if (res != SZ_OK || finished)
return 0;
continue;
}
if (mtc->writeRes != SZ_OK)
res = mtc->writeRes;
for (;;)
{
if (res == SZ_OK && bufIndex != (unsigned)(int)-1)
{
res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex);
if (res != SZ_OK)
{
mtc->writeRes = res;
MtProgress_SetError(&mtc->mtProgress, res);
}
}
if (++wi >= mtc->numBlocksMax)
wi = 0;
{
Bool isReady;
CriticalSection_Enter(&mtc->cs);
if (bufIndex != (unsigned)(int)-1)
{
mtc->freeBlockList[bufIndex] = mtc->freeBlockHead;
mtc->freeBlockHead = bufIndex;
}
isReady = mtc->ReadyBlocks[wi];
if (isReady)
mtc->ReadyBlocks[wi] = False;
else
mtc->writeIndex = wi;
CriticalSection_Leave(&mtc->cs);
RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore))
if (!isReady)
break;
}
{
CMtCoderBlock *block = &mtc->blocks[wi];
if (res == SZ_OK && block->res != SZ_OK)
res = block->res;
bufIndex = block->bufIndex;
finished = block->finished;
}
}
}
#endif
if (finished || res != SZ_OK)
return 0;
}
}
void MtCoder_Construct(CMtCoder* p)
static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp)
{
CMtCoderThread *t = (CMtCoderThread *)pp;
for (;;)
{
if (Event_Wait(&t->startEvent) != 0)
return SZ_ERROR_THREAD;
if (t->stop)
return 0;
{
SRes res = ThreadFunc2(t);
CMtCoder *mtc = t->mtCoder;
if (res != SZ_OK)
{
MtProgress_SetError(&mtc->mtProgress, res);
}
#ifndef MTCODER__USE_WRITE_THREAD
{
unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads);
if (numFinished == mtc->numStartedThreads)
if (Event_Set(&mtc->finishedEvent) != 0)
return SZ_ERROR_THREAD;
}
#endif
}
}
}
void MtCoder_Construct(CMtCoder *p)
{
unsigned i;
p->alloc = 0;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
p->blockSize = 0;
p->numThreadsMax = 0;
p->expectedDataSize = (UInt64)(Int64)-1;
p->inStream = NULL;
p->inData = NULL;
p->inDataSize = 0;
p->progress = NULL;
p->allocBig = NULL;
p->mtCallback = NULL;
p->mtCallbackObject = NULL;
p->allocatedBufsSize = 0;
Event_Construct(&p->readEvent);
Semaphore_Construct(&p->blocksSemaphore);
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CMtThread *t = &p->threads[i];
CMtCoderThread *t = &p->threads[i];
t->mtCoder = p;
t->index = i;
CMtThread_Construct(t, p);
t->inBuf = NULL;
t->stop = False;
Event_Construct(&t->startEvent);
Thread_Construct(&t->thread);
}
#ifdef MTCODER__USE_WRITE_THREAD
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
Event_Construct(&p->writeEvents[i]);
#else
Event_Construct(&p->finishedEvent);
#endif
CriticalSection_Init(&p->cs);
CriticalSection_Init(&p->mtProgress.cs);
}
void MtCoder_Destruct(CMtCoder* p)
static void MtCoder_Free(CMtCoder *p)
{
unsigned i;
for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++)
CMtThread_Destruct(&p->threads[i]);
/*
p->stopReading = True;
if (Event_IsCreated(&p->readEvent))
Event_Set(&p->readEvent);
*/
for (i = 0; i < MTCODER__THREADS_MAX; i++)
MtCoderThread_Destruct(&p->threads[i]);
Event_Close(&p->readEvent);
Semaphore_Close(&p->blocksSemaphore);
#ifdef MTCODER__USE_WRITE_THREAD
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
Event_Close(&p->writeEvents[i]);
#else
Event_Close(&p->finishedEvent);
#endif
}
void MtCoder_Destruct(CMtCoder *p)
{
MtCoder_Free(p);
CriticalSection_Delete(&p->cs);
CriticalSection_Delete(&p->mtProgress.cs);
}
SRes MtCoder_Code(CMtCoder *p)
{
unsigned i, numThreads = p->numThreads;
unsigned numThreads = p->numThreadsMax;
unsigned numBlocksMax;
unsigned i;
SRes res = SZ_OK;
p->res = SZ_OK;
if (numThreads > MTCODER__THREADS_MAX)
numThreads = MTCODER__THREADS_MAX;
numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads);
if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++;
if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++;
if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++;
if (numBlocksMax > MTCODER__BLOCKS_MAX)
numBlocksMax = MTCODER__BLOCKS_MAX;
if (p->blockSize != p->allocatedBufsSize)
{
for (i = 0; i < MTCODER__THREADS_MAX; i++)
{
CMtCoderThread *t = &p->threads[i];
if (t->inBuf)
{
ISzAlloc_Free(p->allocBig, t->inBuf);
t->inBuf = NULL;
}
}
p->allocatedBufsSize = p->blockSize;
}
p->readRes = SZ_OK;
MtProgress_Init(&p->mtProgress, p->progress);
for (i = 0; i < numThreads; i++)
#ifdef MTCODER__USE_WRITE_THREAD
for (i = 0; i < numBlocksMax; i++)
{
RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i]));
}
#else
RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent));
#endif
{
RINOK(CMtThread_Prepare(&p->threads[i]));
RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent));
if (Semaphore_IsCreated(&p->blocksSemaphore))
{
RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore));
}
RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax));
}
for (i = 0; i < numThreads; i++)
for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++)
p->freeBlockList[i] = i + 1;
p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1;
p->freeBlockHead = 0;
p->readProcessed = 0;
p->blockIndex = 0;
p->numBlocksMax = numBlocksMax;
p->stopReading = False;
#ifndef MTCODER__USE_WRITE_THREAD
p->writeIndex = 0;
p->writeRes = SZ_OK;
for (i = 0; i < MTCODER__BLOCKS_MAX; i++)
p->ReadyBlocks[i] = False;
p->numFinishedThreads = 0;
#endif
p->numStartedThreadsLimit = numThreads;
p->numStartedThreads = 0;
// for (i = 0; i < numThreads; i++)
{
CMtThread *t = &p->threads[i];
CLoopThread *lt = &t->thread;
CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++];
RINOK(MtCoderThread_CreateAndStart(nextThread));
}
if (!Thread_WasCreated(&lt->thread))
RINOK_THREAD(Event_Set(&p->readEvent))
#ifdef MTCODER__USE_WRITE_THREAD
{
unsigned bi = 0;
for (;; bi++)
{
lt->func = ThreadFunc;
lt->param = t;
if (bi >= numBlocksMax)
bi = 0;
RINOK_THREAD(Event_Wait(&p->writeEvents[bi]))
if (LoopThread_Create(lt) != SZ_OK)
{
res = SZ_ERROR_THREAD;
break;
const CMtCoderBlock *block = &p->blocks[bi];
unsigned bufIndex = block->bufIndex;
Bool finished = block->finished;
if (res == SZ_OK && block->res != SZ_OK)
res = block->res;
if (bufIndex != (unsigned)(int)-1)
{
if (res == SZ_OK)
{
res = p->mtCallback->Write(p->mtCallbackObject, bufIndex);
if (res != SZ_OK)
MtProgress_SetError(&p->mtProgress, res);
}
CriticalSection_Enter(&p->cs);
{
p->freeBlockList[bufIndex] = p->freeBlockHead;
p->freeBlockHead = bufIndex;
}
CriticalSection_Leave(&p->cs);
}
RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore))
if (finished)
break;
}
}
}
#else
{
WRes wres = Event_Wait(&p->finishedEvent);
res = MY_SRes_HRESULT_FROM_WRes(wres);
}
#endif
if (res == SZ_OK)
{
unsigned j;
for (i = 0; i < numThreads; i++)
{
CMtThread *t = &p->threads[i];
if (LoopThread_StartSubThread(&t->thread) != SZ_OK)
{
res = SZ_ERROR_THREAD;
p->threads[0].stopReading = True;
break;
}
}
res = p->readRes;
Event_Set(&p->threads[0].canWrite);
Event_Set(&p->threads[0].canRead);
if (res == SZ_OK)
res = p->mtProgress.res;
for (j = 0; j < i; j++)
LoopThread_WaitSubThread(&p->threads[j].thread);
}
#ifndef MTCODER__USE_WRITE_THREAD
if (res == SZ_OK)
res = p->writeRes;
#endif
for (i = 0; i < numThreads; i++)
CMtThread_CloseEvents(&p->threads[i]);
return (res == SZ_OK) ? p->res : res;
if (res != SZ_OK)
MtCoder_Free(p);
return res;
}

155
C/MtCoder.h
View File

@@ -1,5 +1,5 @@
/* MtCoder.h -- Multi-thread Coder
2017-04-03 : Igor Pavlov : Public domain */
2017-06-18 : Igor Pavlov : Public domain */
#ifndef __MT_CODER_H
#define __MT_CODER_H
@@ -8,95 +8,144 @@
EXTERN_C_BEGIN
typedef struct
{
CThread thread;
CAutoResetEvent startEvent;
CAutoResetEvent finishedEvent;
int stop;
THREAD_FUNC_TYPE func;
LPVOID param;
THREAD_FUNC_RET_TYPE res;
} CLoopThread;
void LoopThread_Construct(CLoopThread *p);
void LoopThread_Close(CLoopThread *p);
WRes LoopThread_Create(CLoopThread *p);
WRes LoopThread_StopAndWait(CLoopThread *p);
WRes LoopThread_StartSubThread(CLoopThread *p);
WRes LoopThread_WaitSubThread(CLoopThread *p);
/*
if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream
if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream
*/
/* #define MTCODER__USE_WRITE_THREAD */
#ifndef _7ZIP_ST
#define NUM_MT_CODER_THREADS_MAX 32
#define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1)
#define MTCODER__THREADS_MAX 64
#define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3)
#else
#define NUM_MT_CODER_THREADS_MAX 1
#define MTCODER__THREADS_MAX 1
#define MTCODER__BLOCKS_MAX 1
#endif
typedef struct
{
UInt64 inSize;
UInt64 outSize;
} CMtProgressSizes;
typedef struct
{
UInt64 totalInSize;
UInt64 totalOutSize;
ICompressProgress *progress;
SRes res;
UInt64 totalInSize;
UInt64 totalOutSize;
CCriticalSection cs;
UInt64 inSizes[NUM_MT_CODER_THREADS_MAX];
UInt64 outSizes[NUM_MT_CODER_THREADS_MAX];
CMtProgressSizes sizes[MTCODER__THREADS_MAX];
} CMtProgress;
SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize);
typedef struct
{
ICompressProgress vt;
CMtProgress *mtProgress;
unsigned index;
} CMtProgressThunk;
void MtProgressThunk_CreateVTable(CMtProgressThunk *p);
struct _CMtCoder;
typedef struct
{
struct _CMtCoder *mtCoder;
Byte *outBuf;
size_t outBufSize;
Byte *inBuf;
size_t inBufSize;
unsigned index;
CLoopThread thread;
int stop;
Byte *inBuf;
Bool stopReading;
Bool stopWriting;
CAutoResetEvent canRead;
CAutoResetEvent canWrite;
} CMtThread;
CAutoResetEvent startEvent;
CThread thread;
} CMtCoderThread;
typedef struct IMtCoderCallback IMtCoderCallback;
struct IMtCoderCallback
typedef struct
{
SRes (*Code)(const IMtCoderCallback *p, unsigned index, Byte *dest, size_t *destSize,
SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex,
const Byte *src, size_t srcSize, int finished);
};
#define IMtCoderCallback_Code(p, index, dest, destSize, src, srcSize, finished) (p)->Code(p, index, dest, destSize, src, srcSize, finished)
SRes (*Write)(void *p, unsigned outBufIndex);
} IMtCoderCallback2;
typedef struct
{
SRes res;
unsigned bufIndex;
Bool finished;
} CMtCoderBlock;
typedef struct _CMtCoder
{
size_t blockSize;
size_t destBlockSize;
unsigned numThreads;
/* input variables */
ISeqInStream *inStream;
ISeqOutStream *outStream;
ICompressProgress *progress;
ISzAllocPtr alloc;
size_t blockSize; /* size of input block */
unsigned numThreadsMax;
UInt64 expectedDataSize;
ISeqInStream *inStream;
const Byte *inData;
size_t inDataSize;
ICompressProgress *progress;
ISzAllocPtr allocBig;
IMtCoderCallback2 *mtCallback;
void *mtCallbackObject;
/* internal variables */
size_t allocatedBufsSize;
CAutoResetEvent readEvent;
CSemaphore blocksSemaphore;
Bool stopReading;
SRes readRes;
#ifdef MTCODER__USE_WRITE_THREAD
CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX];
#else
CAutoResetEvent finishedEvent;
SRes writeRes;
unsigned writeIndex;
Byte ReadyBlocks[MTCODER__BLOCKS_MAX];
LONG numFinishedThreads;
#endif
unsigned numStartedThreadsLimit;
unsigned numStartedThreads;
unsigned numBlocksMax;
unsigned blockIndex;
UInt64 readProcessed;
IMtCoderCallback *mtCallback;
CCriticalSection cs;
SRes res;
unsigned freeBlockHead;
unsigned freeBlockList[MTCODER__BLOCKS_MAX];
CMtProgress mtProgress;
CMtThread threads[NUM_MT_CODER_THREADS_MAX];
CMtCoderBlock blocks[MTCODER__BLOCKS_MAX];
CMtCoderThread threads[MTCODER__THREADS_MAX];
} CMtCoder;
void MtCoder_Construct(CMtCoder* p);
void MtCoder_Destruct(CMtCoder* p);
void MtCoder_Construct(CMtCoder *p);
void MtCoder_Destruct(CMtCoder *p);
SRes MtCoder_Code(CMtCoder *p);
EXTERN_C_END
#endif

14
C/Threads.c
View File

@@ -1,5 +1,5 @@
/* Threads.c -- multithreading library
2014-09-21 : Igor Pavlov : Public domain */
2017-06-26 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -12,18 +12,20 @@
static WRes GetError()
{
DWORD res = GetLastError();
return (res) ? (WRes)(res) : 1;
return res ? (WRes)res : 1;
}
WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); }
WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); }
static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); }
WRes HandlePtr_Close(HANDLE *p)
{
if (*p != NULL)
{
if (!CloseHandle(*p))
return GetError();
*p = NULL;
*p = NULL;
}
return 0;
}
@@ -49,7 +51,7 @@ WRes Thread_Create(CThread *p, THREAD_FUNC_TYPE func, LPVOID param)
return HandleToWRes(*p);
}
WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled)
{
*p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL);
return HandleToWRes(*p);

5
C/Threads.h
View File

@@ -1,5 +1,5 @@
/* Threads.h -- multithreading library
2013-11-12 : Igor Pavlov : Public domain */
2017-06-18 : Igor Pavlov : Public domain */
#ifndef __7Z_THREADS_H
#define __7Z_THREADS_H
@@ -49,7 +49,8 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled);
WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p);
typedef HANDLE CSemaphore;
#define Semaphore_Construct(p) (*p) = NULL
#define Semaphore_Construct(p) *(p) = NULL
#define Semaphore_IsCreated(p) (*(p) != NULL)
#define Semaphore_Close(p) HandlePtr_Close(p)
#define Semaphore_Wait(p) Handle_WaitObject(*(p))
WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount);

11
C/Util/7z/7zMain.c
View File

@@ -1,5 +1,5 @@
/* 7zMain.c - Test application for 7z Decoder
2017-04-05 : Igor Pavlov : Public domain */
2017-08-26 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -616,7 +616,14 @@ int MY_CDECL main(int numargs, char *args[])
#ifdef USE_WINDOWS_FILE
if (SzBitWithVals_Check(&db.Attribs, i))
SetFileAttributesW(destPath, db.Attribs.Vals[i]);
{
UInt32 attrib = db.Attribs.Vals[i];
/* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker.
We remove posix bits, if we detect posix mode field */
if ((attrib & 0xF0000000) != 0)
attrib &= 0x7FFF;
SetFileAttributesW(destPath, attrib);
}
#endif
}
PrintLF();

4
C/Util/7zipInstall/7zipInstall.c
View File

@@ -1,5 +1,5 @@
/* 7zipInstall.c - 7-Zip Installer
2017-04-04 : Igor Pavlov : Public domain */
2017-08-28 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -154,7 +154,7 @@ static WRes MyCreateDir(LPCWSTR name)
}
#define IS_SEPAR(c) (c == WCHAR_PATH_SEPARATOR)
#define IS_LETTER_CHAR(c) ((c) >= 'a' && (c) <= 'z' || (c) >= 'A' && (c) <= 'Z')
#define IS_LETTER_CHAR(c) (((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
#define IS_DRIVE_PATH(s) (IS_LETTER_CHAR(s[0]) && s[1] == ':' && IS_SEPAR(s[2]))
static int ReverseFind_PathSepar(const wchar_t *s)

8
C/Util/Lzma/LzmaUtil.dsp
View File

@@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c
# End Source File
# Begin Source File
SOURCE=..\..\7zTypes.h
# End Source File
# Begin Source File
SOURCE=..\..\7zVersion.h
# End Source File
# Begin Source File
@@ -160,9 +164,5 @@ SOURCE=..\..\Threads.c
SOURCE=..\..\Threads.h
# End Source File
# Begin Source File
SOURCE=..\..\Types.h
# End Source File
# End Target
# End Project

12
C/Xz.c
View File

@@ -1,5 +1,5 @@
/* Xz.c - Xz
2017-04-03 : Igor Pavlov : Public domain */
2017-05-12 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -9,7 +9,7 @@
#include "XzCrc64.h"
const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 };
const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' };
/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */
unsigned Xz_WriteVarInt(Byte *buf, UInt64 v)
{
@@ -26,16 +26,16 @@ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v)
void Xz_Construct(CXzStream *p)
{
p->numBlocks = p->numBlocksAllocated = 0;
p->blocks = 0;
p->numBlocks = 0;
p->blocks = NULL;
p->flags = 0;
}
void Xz_Free(CXzStream *p, ISzAllocPtr alloc)
{
ISzAlloc_Free(alloc, p->blocks);
p->numBlocks = p->numBlocksAllocated = 0;
p->blocks = 0;
p->numBlocks = 0;
p->blocks = NULL;
}
unsigned XzFlags_GetCheckSize(CXzStreamFlags f)

35
C/Xz.h
View File

@@ -1,5 +1,5 @@
/* Xz.h - Xz interface
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#ifndef __XZ_H
#define __XZ_H
@@ -50,6 +50,7 @@ typedef struct
#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1)
#define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0)
#define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0)
#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0)
SRes XzBlock_Parse(CXzBlock *p, const Byte *header);
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes);
@@ -60,7 +61,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
#define XZ_FOOTER_SIG_SIZE 2
extern const Byte XZ_SIG[XZ_SIG_SIZE];
/*
extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE];
*/
#define XZ_FOOTER_SIG_0 'Y'
#define XZ_FOOTER_SIG_1 'Z'
#define XZ_STREAM_FLAGS_SIZE 2
#define XZ_STREAM_CRC_SIZE 4
@@ -106,7 +113,6 @@ typedef struct
{
CXzStreamFlags flags;
size_t numBlocks;
size_t numBlocksAllocated;
CXzBlockSizes *blocks;
UInt64 startOffset;
} CXzStream;
@@ -218,6 +224,9 @@ typedef struct
CXzBlock block;
CXzCheck check;
CSha256 sha;
unsigned decodeOnlyOneBlock;
Byte shaDigest[SHA256_DIGEST_SIZE];
Byte buf[XZ_BLOCK_HEADER_SIZE_MAX];
} CXzUnpacker;
@@ -258,7 +267,7 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode,
ECoderStatus *status);
Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p);
Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p);
/*
Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of
@@ -268,7 +277,25 @@ XzUnpacker_Code() returns:
res == SZ_ERROR_NO_ARCHIVE
*/
UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p);
UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p);
/*
for random block decoding:
XzUnpacker_Init();
set CXzUnpacker::streamFlags
XzUnpacker_PrepareToRandomBlockDecoding()
loop
{
XzUnpacker_Code()
XzUnpacker_IsBlockFinished()
}
*/
void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p);
Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p);
#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags))
EXTERN_C_END

16
C/XzCrc64.c
View File

@@ -1,5 +1,5 @@
/* XzCrc64.c -- CRC64 calculation
2017-04-03 : Igor Pavlov : Public domain */
2017-05-23 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -9,9 +9,9 @@
#define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42)
#ifdef MY_CPU_LE
#define CRC_NUM_TABLES 4
#define CRC64_NUM_TABLES 4
#else
#define CRC_NUM_TABLES 5
#define CRC64_NUM_TABLES 5
#define CRC_UINT64_SWAP(v) \
((v >> 56) \
| ((v >> 40) & ((UInt64)0xFF << 8)) \
@@ -29,10 +29,10 @@
UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table);
#endif
typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table);
typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table);
static CRC_FUNC g_Crc64Update;
UInt64 g_Crc64Table[256 * CRC_NUM_TABLES];
static CRC64_FUNC g_Crc64Update;
UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES];
UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size)
{
@@ -55,7 +55,7 @@ void MY_FAST_CALL Crc64GenerateTable()
r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1)));
g_Crc64Table[i] = r;
}
for (i = 256; i < 256 * CRC_NUM_TABLES; i++)
for (i = 256; i < 256 * CRC64_NUM_TABLES; i++)
{
UInt64 r = g_Crc64Table[(size_t)i - 256];
g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8);
@@ -74,7 +74,7 @@ void MY_FAST_CALL Crc64GenerateTable()
else
#endif
{
for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--)
for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--)
{
UInt64 x = g_Crc64Table[(size_t)i - 256];
g_Crc64Table[i] = CRC_UINT64_SWAP(x);

14
C/XzCrc64Opt.c
View File

@@ -1,5 +1,5 @@
/* XzCrc64Opt.c -- CRC64 calculation
2017-04-03 : Igor Pavlov : Public domain */
2017-06-30 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -7,13 +7,13 @@
#ifndef MY_CPU_BE
#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table)
{
const Byte *p = (const Byte *)data;
for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++)
v = CRC_UPDATE_BYTE_2(v, *p);
v = CRC64_UPDATE_BYTE_2(v, *p);
for (; size >= 4; size -= 4, p += 4)
{
UInt32 d = (UInt32)v ^ *(const UInt32 *)p;
@@ -24,7 +24,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con
^ (table + 0x000)[((d >> 24))];
}
for (; size > 0; size--, p++)
v = CRC_UPDATE_BYTE_2(v, *p);
v = CRC64_UPDATE_BYTE_2(v, *p);
return v;
}
@@ -43,7 +43,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, con
| ((v << 40) & ((UInt64)0xFF << 48)) \
| ((v << 56)))
#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8))
#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8))
UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table)
{
@@ -51,7 +51,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size
table += 0x100;
v = CRC_UINT64_SWAP(v);
for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++)
v = CRC_UPDATE_BYTE_2_BE(v, *p);
v = CRC64_UPDATE_BYTE_2_BE(v, *p);
for (; size >= 4; size -= 4, p += 4)
{
UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p;
@@ -62,7 +62,7 @@ UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size
^ (table + 0x300)[((d >> 24))];
}
for (; size > 0; size--, p++)
v = CRC_UPDATE_BYTE_2_BE(v, *p);
v = CRC64_UPDATE_BYTE_2_BE(v, *p);
return CRC_UINT64_SWAP(v);
}

136
C/XzDec.c
View File

@@ -1,5 +1,5 @@
/* XzDec.c -- Xz Decode
2017-04-03 : Igor Pavlov : Public domain */
2017-07-27 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -393,7 +393,7 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
{
SizeT destLenOrig = *destLen;
SizeT srcLenOrig = *srcLen;
Bool allFinished = True;
*destLen = 0;
*srcLen = 0;
*status = CODER_STATUS_NOT_FINISHED;
@@ -411,6 +411,7 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
for (;;)
{
Bool processed = False;
Bool allFinished = True;
unsigned i;
/*
if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY)
@@ -487,14 +488,17 @@ SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen,
if (destLenCur != 0 || srcLenCur != 0)
processed = True;
}
if (!processed)
break;
{
if (allFinished)
*status = CODER_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
}
if (allFinished)
*status = CODER_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf)
{
*p = (CXzStreamFlags)GetBe16(buf + XZ_SIG_SIZE);
@@ -506,11 +510,11 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf)
static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf)
{
return
indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) &&
(GetUi32(buf) == CrcCalc(buf + 4, 6) &&
flags == GetBe16(buf + 8) &&
memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0);
return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2)
&& GetUi32(buf) == CrcCalc(buf + 4, 6)
&& flags == GetBe16(buf + 8)
&& buf[10] == XZ_FOOTER_SIG_0
&& buf[11] == XZ_FOOTER_SIG_1;
}
#define READ_VARINT_AND_CHECK(buf, pos, size, res) \
@@ -524,14 +528,15 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
unsigned numFilters, i;
unsigned headerSize = (unsigned)header[0] << 2;
/* (headerSize != 0) : another code checks */
if (CrcCalc(header, headerSize) != GetUi32(header + headerSize))
return SZ_ERROR_ARCHIVE;
pos = 1;
if (pos == headerSize)
return SZ_ERROR_ARCHIVE;
p->flags = header[pos++];
p->packSize = (UInt64)(Int64)-1;
if (XzBlock_HasPackSize(p))
{
READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize);
@@ -539,6 +544,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
return SZ_ERROR_ARCHIVE;
}
p->unpackSize = (UInt64)(Int64)-1;
if (XzBlock_HasUnpackSize(p))
READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize);
@@ -565,6 +571,9 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
#endif
}
if (XzBlock_HasUnsupportedFlags(p))
return SZ_ERROR_UNSUPPORTED;
while (pos < headerSize)
if (header[pos++] != 0)
return SZ_ERROR_ARCHIVE;
@@ -615,6 +624,7 @@ void XzUnpacker_Init(CXzUnpacker *p)
p->numFinishedStreams = 0;
p->numTotalBlocks = 0;
p->padSize = 0;
p->decodeOnlyOneBlock = 0;
}
void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc)
@@ -628,6 +638,18 @@ void XzUnpacker_Free(CXzUnpacker *p)
MixCoder_Free(&p->decoder);
}
void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p)
{
p->indexSize = 0;
p->numBlocks = 0;
Sha256_Init(&p->sha);
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
p->decodeOnlyOneBlock = 1;
}
SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status)
{
@@ -638,20 +660,44 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
*status = CODER_STATUS_NOT_SPECIFIED;
for (;;)
{
SizeT srcRem = srcLenOrig - *srcLen;
SizeT srcRem;
if (p->state == XZ_STATE_BLOCK)
{
SizeT destLen2 = destLenOrig - *destLen;
SizeT srcLen2 = srcLenOrig - *srcLen;
SRes res;
ECoderFinishMode finishMode2 = finishMode;
if (p->block.packSize != (UInt64)(Int64)-1)
{
UInt64 rem = p->block.packSize - p->packSize;
if (srcLen2 > rem)
srcLen2 = (SizeT)rem;
if (rem == 0 && p->block.unpackSize == p->unpackSize)
return SZ_ERROR_DATA;
}
if (p->block.unpackSize != (UInt64)(Int64)-1)
{
UInt64 rem = p->block.unpackSize - p->unpackSize;
if (destLen2 >= rem)
{
finishMode2 = CODER_FINISH_END;
destLen2 = (SizeT)rem;
}
}
/*
if (srcLen2 == 0 && destLen2 == 0)
{
*status = CODER_STATUS_NOT_FINISHED;
return SZ_OK;
}
*/
res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode, status);
res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode2, status);
XzCheck_Update(&p->check, dest, destLen2);
(*srcLen) += srcLen2;
@@ -661,28 +707,41 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
(*destLen) += destLen2;
dest += destLen2;
p->unpackSize += destLen2;
RINOK(res);
if (*status == CODER_STATUS_FINISHED_WITH_MARK)
if (*status != CODER_STATUS_FINISHED_WITH_MARK)
{
if (p->block.packSize == p->packSize
&& *status == CODER_STATUS_NEEDS_MORE_INPUT)
{
*status = CODER_STATUS_NOT_SPECIFIED;
return SZ_ERROR_DATA;
}
// if (srcLen2 == 0 && destLen2 == 0)
return SZ_OK;
}
{
Byte temp[32];
unsigned num = Xz_WriteVarInt(temp, p->packSize + p->blockHeaderSize + XzFlags_GetCheckSize(p->streamFlags));
unsigned num = Xz_WriteVarInt(temp, XzUnpacker_GetPackSizeForIndex(p));
num += Xz_WriteVarInt(temp + num, p->unpackSize);
Sha256_Update(&p->sha, temp, num);
p->indexSize += num;
p->numBlocks++;
p->state = XZ_STATE_BLOCK_FOOTER;
p->pos = 0;
p->alignPos = 0;
if (p->block.unpackSize != (UInt64)(Int64)-1)
if (p->block.unpackSize != p->unpackSize)
return SZ_ERROR_DATA;
}
else if (srcLen2 == 0 && destLen2 == 0)
return SZ_OK;
continue;
// continue;
}
srcRem = srcLenOrig - *srcLen;
if (srcRem == 0)
{
*status = CODER_STATUS_NEEDS_MORE_INPUT;
@@ -704,10 +763,10 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
{
RINOK(Xz_ParseHeader(&p->streamFlags, p->buf));
p->numStartedStreams++;
p->state = XZ_STATE_BLOCK_HEADER;
Sha256_Init(&p->sha);
p->indexSize = 0;
p->numBlocks = 0;
Sha256_Init(&p->sha);
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
}
break;
@@ -721,6 +780,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
(*srcLen)++;
if (p->buf[0] == 0)
{
if (p->decodeOnlyOneBlock)
return SZ_ERROR_DATA;
p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks);
p->indexPos = p->indexPreSize;
p->indexSize += p->indexPreSize;
@@ -728,10 +789,13 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
Sha256_Init(&p->sha);
p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize);
p->state = XZ_STATE_STREAM_INDEX;
break;
}
p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4;
break;
}
else if (p->pos != p->blockHeaderSize)
if (p->pos != p->blockHeaderSize)
{
UInt32 cur = p->blockHeaderSize - p->pos;
if (cur > srcRem)
@@ -775,14 +839,20 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
p->pos += cur;
(*srcLen) += cur;
src += cur;
if (checkSize != p->pos)
break;
}
else
{
Byte digest[XZ_CHECK_SIZE_MAX];
p->state = XZ_STATE_BLOCK_HEADER;
p->pos = 0;
if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0)
return SZ_ERROR_CRC;
if (p->decodeOnlyOneBlock)
{
*status = CODER_STATUS_FINISHED_WITH_MARK;
return SZ_OK;
}
}
}
break;
@@ -898,12 +968,18 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
*/
}
Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p)
Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p)
{
return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0);
}
Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p)
{
return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0);
}
UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p)
UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p)
{
UInt64 num = 0;
if (p->state == XZ_STATE_STREAM_PADDING)

1258
C/XzEnc.c
View File

File diff suppressed because it is too large Load Diff

27
C/XzEnc.h
View File

@@ -1,5 +1,5 @@
/* XzEnc.h -- Xz Encode
2011-02-07 : Igor Pavlov : Public domain */
2017-06-27 : Igor Pavlov : Public domain */
#ifndef __XZ_ENC_H
#define __XZ_ENC_H
@@ -10,6 +10,11 @@
EXTERN_C_BEGIN
#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO
#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID
typedef struct
{
UInt32 id;
@@ -20,15 +25,31 @@ typedef struct
void XzFilterProps_Init(CXzFilterProps *p);
typedef struct
{
const CLzma2EncProps *lzma2Props;
const CXzFilterProps *filterProps;
CLzma2EncProps lzma2Props;
CXzFilterProps filterProps;
unsigned checkId;
UInt64 blockSize;
int numBlockThreads_Reduced;
int numBlockThreads_Max;
int numTotalThreads;
int forceWriteSizesInHeader;
UInt64 reduceSize;
} CXzProps;
void XzProps_Init(CXzProps *p);
typedef void * CXzEncHandle;
CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig);
void XzEnc_Destroy(CXzEncHandle p);
SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props);
void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize);
SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress);
SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
const CXzProps *props, ICompressProgress *progress);

32
C/XzIn.c
View File

@@ -1,5 +1,5 @@
/* XzIn.c - Xz input
2017-04-03 : Igor Pavlov : Public domain */
2017-05-11 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -9,6 +9,12 @@
#include "CpuArch.h"
#include "Xz.h"
/*
#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0)
*/
#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1)
SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream)
{
Byte sig[XZ_STREAM_HEADER_SIZE];
@@ -28,7 +34,7 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
unsigned headerSize;
*headerSizeRes = 0;
RINOK(SeqInStream_ReadByte(inStream, &header[0]));
headerSize = ((unsigned)header[0] << 2) + 4;
headerSize = (unsigned)header[0];
if (headerSize == 0)
{
*headerSizeRes = 1;
@@ -37,12 +43,13 @@ SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt
}
*isIndex = False;
headerSize = (headerSize << 2) + 4;
*headerSizeRes = headerSize;
RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1));
return XzBlock_Parse(p, header);
}
#define ADD_SIZE_CHECH(size, val) \
#define ADD_SIZE_CHECK(size, val) \
{ UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; }
UInt64 Xz_GetUnpackSize(const CXzStream *p)
@@ -50,7 +57,7 @@ UInt64 Xz_GetUnpackSize(const CXzStream *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->numBlocks; i++)
ADD_SIZE_CHECH(size, p->blocks[i].unpackSize);
ADD_SIZE_CHECK(size, p->blocks[i].unpackSize);
return size;
}
@@ -59,7 +66,7 @@ UInt64 Xz_GetPackSize(const CXzStream *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->numBlocks; i++)
ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3);
ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3);
return size;
}
@@ -96,9 +103,8 @@ static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPt
{
size_t i;
p->numBlocks = numBlocks;
p->numBlocksAllocated = numBlocks;
p->blocks = ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks);
if (p->blocks == 0)
if (!p->blocks)
return SZ_ERROR_MEM;
for (i = 0; i < numBlocks; i++)
{
@@ -126,7 +132,7 @@ static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize,
if (size != indexSize)
return SZ_ERROR_UNSUPPORTED;
buf = ISzAlloc_Alloc(alloc, size);
if (buf == 0)
if (!buf)
return SZ_ERROR_MEM;
res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED);
if (res == SZ_OK)
@@ -154,7 +160,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff
pos -= XZ_STREAM_FOOTER_SIZE;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE));
if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0)
if (!XZ_FOOTER_SIG_CHECK(buf + 10))
{
UInt32 total = 0;
pos += XZ_STREAM_FOOTER_SIZE;
@@ -187,7 +193,7 @@ static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOff
return SZ_ERROR_NO_ARCHIVE;
pos -= XZ_STREAM_FOOTER_SIZE;
RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE));
if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0)
if (!XZ_FOOTER_SIG_CHECK(buf + 10))
return SZ_ERROR_NO_ARCHIVE;
}
@@ -262,7 +268,7 @@ UInt64 Xzs_GetUnpackSize(const CXzs *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->num; i++)
ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i]));
ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i]));
return size;
}
@@ -272,7 +278,7 @@ UInt64 Xzs_GetPackSize(const CXzs *p)
UInt64 size = 0;
size_t i;
for (i = 0; i < p->num; i++)
ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i]));
ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i]));
return size;
}
*/
@@ -294,7 +300,7 @@ SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompr
{
size_t newNum = p->num + p->num / 4 + 1;
Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream));
if (data == 0)
if (!data)
return SZ_ERROR_MEM;
p->numAllocated = newNum;
if (p->num != 0)