xevion/easy7zip
1
0
Fork 0
mirror of https://github.com/Xevion/easy7zip.git synced 2025-12-06 21:14:58 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'pr/1'

Browse Source
This commit is contained in:
conor42
2019-03-18 00:21:14 +10:00
parent d85962e654 2fdf03cb4a
commit 26cd887a90
261 changed files with 8112 additions and 8922 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
C/7zArcIn.c
View File

@@ -1,5 +1,5 @@
/* 7zArcIn.c -- 7z Input functions
2017-04-03 : Igor Pavlov : Public domain */
2018-12-31 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -19,7 +19,7 @@
{ MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); }
#define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \
{ if ((size) == 0) p = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } }
{ if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } }
#define k7zMajorVersion 0
@@ -666,7 +666,7 @@ static SRes ReadUnpackInfo(CSzAr *p,
MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc);
MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc);
MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc);
MY_ALLOC(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc);
MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc);
startBufPtr = sd.Data;
@@ -1744,7 +1744,7 @@ size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex)
UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest)
{
Bool needSlash;
BoolInt needSlash;
if (!p->FileNameOffsets)
{
*(--dest) = 0;

12
C/7zDec.c
View File

@@ -1,5 +1,5 @@
/* 7zDec.c -- Decoding from 7z folder
2017-04-03 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -44,7 +44,7 @@ typedef struct
const Byte *end;
const Byte *begin;
UInt64 processed;
Bool extra;
BoolInt extra;
SRes res;
const ILookInStream *inStream;
} CByteInToLook;
@@ -156,7 +156,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I
{
SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos;
ELzmaStatus status;
res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status);
res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status);
lookahead -= inProcessed;
inSize -= inProcessed;
if (res != SZ_OK)
@@ -218,7 +218,7 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize,
{
SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos;
ELzmaStatus status;
res = Lzma2Dec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status);
res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status);
lookahead -= inProcessed;
inSize -= inProcessed;
if (res != SZ_OK)
@@ -269,7 +269,7 @@ static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer
return SZ_OK;
}
static Bool IS_MAIN_METHOD(UInt32 m)
static BoolInt IS_MAIN_METHOD(UInt32 m)
{
switch (m)
{
@@ -286,7 +286,7 @@ static Bool IS_MAIN_METHOD(UInt32 m)
return False;
}
static Bool IS_SUPPORTED_CODER(const CSzCoderInfo *c)
static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c)
{
return
c->NumStreams == 1

5
C/7zTypes.h
View File

@@ -1,5 +1,5 @@
/* 7zTypes.h -- Basic types
2017-07-17 : Igor Pavlov : Public domain */
2018-08-04 : Igor Pavlov : Public domain */
#ifndef __7Z_TYPES_H
#define __7Z_TYPES_H
@@ -103,7 +103,8 @@ typedef UInt32 SizeT;
typedef size_t SizeT;
#endif
typedef int Bool;
typedef int BoolInt;
/* typedef BoolInt Bool; */
#define True 1
#define False 0

10
C/7zVersion.h
View File

@@ -1,7 +1,7 @@
#define MY_VER_MAJOR 18
#define MY_VER_MINOR 05
#define MY_VER_MAJOR 19
#define MY_VER_MINOR 00
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "18.05 ZS v1.3.7 R3"
#define MY_VERSION_NUMBERS "19.00 ZS v1.3.8 R2"
#define MY_VERSION MY_VERSION_NUMBERS
#ifdef MY_CPU_NAME
@@ -10,12 +10,12 @@
#define MY_VERSION_CPU MY_VERSION
#endif
#define MY_DATE "2018-10-28"
#define MY_DATE "2019-03-05"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt"
#define MY_COPYRIGHT_PD "Igor Pavlov : Public domain"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov, 2016-2018 Tino Reichardt"
#define MY_COPYRIGHT_CR "Copyright (c) 1999-2019 Igor Pavlov, 2016-2019 Tino Reichardt"
#ifdef USE_COPYRIGHT_CR
#define MY_COPYRIGHT MY_COPYRIGHT_CR

23
C/7zVersionTr.h
View File

@@ -1,11 +1,14 @@
#define MY_VER_MAJOR 18
#define MY_VER_MINOR 05
#define MY_VER_BUILD 0
#define MY_VERSION_NUMBERS "1.3.7 R3"
#define MY_VERSION MY_VERSION_NUMBERS
#define MY_DATE "2018-10-28"
#undef MY_COPYRIGHT
#undef MY_VERSION_COPYRIGHT_DATE
#include "7zVersion.h"
#undef MY_AUTHOR_NAME
#define MY_AUTHOR_NAME "Tino Reichardt"
#define MY_COPYRIGHT "Copyright (c) 2016 - 2018 Tino Reichardt"
#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " : " MY_COPYRIGHT " : " MY_DATE
#undef MY_COPYRIGHT
#define MY_COPYRIGHT "Copyright (c) 2016 - 2019 Tino Reichardt"
#undef MY_COPYRIGHT_DATE
#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE
#undef MY_VERSION_COPYRIGHT_DATE
#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE

8
C/Bcj2Enc.c
View File

@@ -1,5 +1,5 @@
/* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code)
2017-04-28 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -52,7 +52,7 @@ void Bcj2Enc_Init(CBcj2Enc *p)
p->probs[i] = kBitModelTotal >> 1;
}
static Bool MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p)
static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p)
{
if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0)
{
@@ -165,7 +165,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p)
{
Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]);
Bool needConvert;
BoolInt needConvert;
p->bufs[BCJ2_STREAM_MAIN] = dest + 1;
p->ip += (UInt32)num + 1;
@@ -253,7 +253,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p)
{
const Byte *src = p->src;
const Byte *srcLim = p->srcLim;
unsigned finishMode = p->finishMode;
EBcj2Enc_FinishMode finishMode = p->finishMode;
p->src = p->temp;
p->srcLim = p->temp + p->tempPos;

4
C/BwtSort.c
View File

@@ -1,5 +1,5 @@
/* BwtSort.c -- BWT block sorting
2017-04-03 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -448,7 +448,7 @@ UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize)
groupSize = ((Indices[i] & ~0xC0000000) >> kNumBitsMax);
{
Bool finishedGroup = ((Indices[i] & 0x80000000) == 0);
BoolInt finishedGroup = ((Indices[i] & 0x80000000) == 0);
if ((Indices[i] & 0x40000000) != 0)
{
groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits);

28
C/CpuArch.c
View File

@@ -1,5 +1,5 @@
/* CpuArch.c -- CPU specific code
2016-02-25: Igor Pavlov : Public domain */
2018-02-18: Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -115,7 +115,7 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d)
#endif
}
Bool x86cpuid_CheckAndRead(Cx86cpuid *p)
BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p)
{
CHECK_CPUID_IS_SUPPORTED
MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]);
@@ -144,7 +144,7 @@ int x86cpuid_GetFirm(const Cx86cpuid *p)
return -1;
}
Bool CPU_Is_InOrder()
BoolInt CPU_Is_InOrder()
{
Cx86cpuid p;
int firm;
@@ -175,7 +175,7 @@ Bool CPU_Is_InOrder()
#if !defined(MY_CPU_AMD64) && defined(_WIN32)
#include <windows.h>
static Bool CPU_Sys_Is_SSE_Supported()
static BoolInt CPU_Sys_Is_SSE_Supported()
{
OSVERSIONINFO vi;
vi.dwOSVersionInfoSize = sizeof(vi);
@@ -188,7 +188,7 @@ static Bool CPU_Sys_Is_SSE_Supported()
#define CHECK_SYS_SSE_SUPPORT
#endif
Bool CPU_Is_Aes_Supported()
BoolInt CPU_Is_Aes_Supported()
{
Cx86cpuid p;
CHECK_SYS_SSE_SUPPORT
@@ -197,4 +197,22 @@ Bool CPU_Is_Aes_Supported()
return (p.c >> 25) & 1;
}
BoolInt CPU_IsSupported_PageGB()
{
Cx86cpuid cpuid;
if (!x86cpuid_CheckAndRead(&cpuid))
return False;
{
UInt32 d[4] = { 0 };
MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]);
if (d[0] < 0x80000001)
return False;
}
{
UInt32 d[4] = { 0 };
MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]);
return (d[3] >> 26) & 1;
}
}
#endif

9
C/CpuArch.h
View File

@@ -1,5 +1,5 @@
/* CpuArch.h -- CPU specific code
2017-09-04 : Igor Pavlov : Public domain */
2018-02-18 : Igor Pavlov : Public domain */
#ifndef __CPU_ARCH_H
#define __CPU_ARCH_H
@@ -318,15 +318,16 @@ enum
void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d);
Bool x86cpuid_CheckAndRead(Cx86cpuid *p);
BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p);
int x86cpuid_GetFirm(const Cx86cpuid *p);
#define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF))
#define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF))
#define x86cpuid_GetStepping(ver) (ver & 0xF)
Bool CPU_Is_InOrder();
Bool CPU_Is_Aes_Supported();
BoolInt CPU_Is_InOrder();
BoolInt CPU_Is_Aes_Supported();
BoolInt CPU_IsSupported_PageGB();
#endif

25
C/DllSecur.c
View File

@@ -1,5 +1,5 @@
/* DllSecur.c -- DLL loading security
2016-10-04 : Igor Pavlov : Public domain */
2018-02-21 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -28,10 +28,31 @@ static const char * const g_Dlls =
"CRYPTBASE\0"
"OLEACC\0"
"CLBCATQ\0"
"VERSION\0"
;
#endif
void My_SetDefaultDllDirectories()
{
#ifndef UNDER_CE
OSVERSIONINFO vi;
vi.dwOSVersionInfoSize = sizeof(vi);
GetVersionEx(&vi);
if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0)
{
Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories)
GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories");
if (setDllDirs)
if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS))
return;
}
#endif
}
void LoadSecurityDlls()
{
#ifndef UNDER_CE
@@ -70,7 +91,7 @@ void LoadSecurityDlls()
for (;;)
{
char c = *dll++;
buf[pos + k] = c;
buf[pos + k] = (Byte)c;
k++;
if (c == 0)
break;

3
C/DllSecur.h
View File

@@ -1,5 +1,5 @@
/* DllSecur.h -- DLL loading for security
2016-06-08 : Igor Pavlov : Public domain */
2018-02-19 : Igor Pavlov : Public domain */
#ifndef __DLL_SECUR_H
#define __DLL_SECUR_H
@@ -10,6 +10,7 @@ EXTERN_C_BEGIN
#ifdef _WIN32
void My_SetDefaultDllDirectories();
void LoadSecurityDlls();
#endif

140
C/LzFind.c
View File

@@ -1,5 +1,5 @@
/* LzFind.c -- Match finder for LZ algorithms
2017-06-10 : Igor Pavlov : Public domain */
2018-07-08 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -138,7 +138,7 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p)
void MatchFinder_Construct(CMatchFinder *p)
{
UInt32 i;
unsigned i;
p->bufferBase = NULL;
p->directInput = 0;
p->hash = NULL;
@@ -147,7 +147,7 @@ void MatchFinder_Construct(CMatchFinder *p)
for (i = 0; i < 256; i++)
{
UInt32 r = i;
UInt32 r = (UInt32)i;
unsigned j;
for (j = 0; j < 8; j++)
r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1)));
@@ -368,6 +368,8 @@ static void MatchFinder_Normalize(CMatchFinder *p)
MatchFinder_ReduceOffsets(p, subValue);
}
MY_NO_INLINE
static void MatchFinder_CheckLimits(CMatchFinder *p)
{
if (p->pos == kMaxValForNormalize)
@@ -379,10 +381,16 @@ static void MatchFinder_CheckLimits(CMatchFinder *p)
MatchFinder_SetLimits(p);
}
static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son,
/*
(lenLimit > maxLen)
*/
MY_FORCE_INLINE
static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son,
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue,
UInt32 *distances, UInt32 maxLen)
UInt32 *distances, unsigned maxLen)
{
/*
son[_cyclicBufferPos] = curMatch;
for (;;)
{
@@ -400,7 +408,8 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos,
break;
if (maxLen < len)
{
*distances++ = maxLen = len;
maxLen = len;
*distances++ = len;
*distances++ = delta - 1;
if (len == lenLimit)
return distances;
@@ -408,15 +417,58 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos,
}
}
}
*/
const Byte *lim = cur + lenLimit;
son[_cyclicBufferPos] = curMatch;
do
{
UInt32 delta = pos - curMatch;
if (delta >= _cyclicBufferSize)
break;
{
ptrdiff_t diff;
curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)];
diff = (ptrdiff_t)0 - delta;
if (cur[maxLen] == cur[maxLen + diff])
{
const Byte *c = cur;
while (*c == c[diff])
{
if (++c == lim)
{
distances[0] = (UInt32)(lim - cur);
distances[1] = delta - 1;
return distances + 2;
}
}
{
unsigned len = (unsigned)(c - cur);
if (maxLen < len)
{
maxLen = len;
distances[0] = (UInt32)len;
distances[1] = delta - 1;
distances += 2;
}
}
}
}
}
while (--cutValue);
return distances;
}
MY_FORCE_INLINE
UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son,
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue,
UInt32 *distances, UInt32 maxLen)
{
CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CLzRef *ptr1 = son + (_cyclicBufferPos << 1);
UInt32 len0 = 0, len1 = 0;
CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1;
CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1);
unsigned len0 = 0, len1 = 0;
for (;;)
{
UInt32 delta = pos - curMatch;
@@ -426,9 +478,10 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt
return distances;
}
{
CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
const Byte *pb = cur - delta;
UInt32 len = (len0 < len1 ? len0 : len1);
unsigned len = (len0 < len1 ? len0 : len1);
UInt32 pair0 = pair[0];
if (pb[len] == cur[len])
{
if (++len != lenLimit && pb[len] == cur[len])
@@ -437,11 +490,12 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt
break;
if (maxLen < len)
{
*distances++ = maxLen = len;
maxLen = (UInt32)len;
*distances++ = (UInt32)len;
*distances++ = delta - 1;
if (len == lenLimit)
{
*ptr1 = pair[0];
*ptr1 = pair0;
*ptr0 = pair[1];
return distances;
}
@@ -468,9 +522,9 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt
static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son,
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue)
{
CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CLzRef *ptr1 = son + (_cyclicBufferPos << 1);
UInt32 len0 = 0, len1 = 0;
CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1;
CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1);
unsigned len0 = 0, len1 = 0;
for (;;)
{
UInt32 delta = pos - curMatch;
@@ -480,9 +534,9 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const
return;
}
{
CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
const Byte *pb = cur - delta;
UInt32 len = (len0 < len1 ? len0 : len1);
unsigned len = (len0 < len1 ? len0 : len1);
if (pb[len] == cur[len])
{
while (++len != lenLimit)
@@ -520,13 +574,13 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const
p->buffer++; \
if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p);
#define MOVE_POS_RET MOVE_POS return offset;
#define MOVE_POS_RET MOVE_POS return (UInt32)offset;
static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; }
#define GET_MATCHES_HEADER2(minLen, ret_op) \
UInt32 lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \
lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \
unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \
lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \
cur = p->buffer;
#define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0)
@@ -535,22 +589,22 @@ static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; }
#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue
#define GET_MATCHES_FOOTER(offset, maxLen) \
offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \
distances + offset, maxLen) - distances); MOVE_POS_RET;
offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \
distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET;
#define SKIP_FOOTER \
SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS;
SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS;
#define UPDATE_maxLen { \
ptrdiff_t diff = (ptrdiff_t)0 - d2; \
const Byte *c = cur + maxLen; \
const Byte *lim = cur + lenLimit; \
for (; c != lim; c++) if (*(c + diff) != *c) break; \
maxLen = (UInt32)(c - cur); }
maxLen = (unsigned)(c - cur); }
static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
{
UInt32 offset;
unsigned offset;
GET_MATCHES_HEADER(2)
HASH2_CALC;
curMatch = p->hash[hv];
@@ -561,7 +615,7 @@ static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
{
UInt32 offset;
unsigned offset;
GET_MATCHES_HEADER(3)
HASH_ZIP_CALC;
curMatch = p->hash[hv];
@@ -572,7 +626,8 @@ UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
{
UInt32 h2, d2, maxLen, offset, pos;
UInt32 h2, d2, pos;
unsigned maxLen, offset;
UInt32 *hash;
GET_MATCHES_HEADER(3)
@@ -594,12 +649,12 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur)
{
UPDATE_maxLen
distances[0] = maxLen;
distances[0] = (UInt32)maxLen;
distances[1] = d2 - 1;
offset = 2;
if (maxLen == lenLimit)
{
SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p));
SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p));
MOVE_POS_RET;
}
}
@@ -609,7 +664,8 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
{
UInt32 h2, h3, d2, d3, maxLen, offset, pos;
UInt32 h2, h3, d2, d3, pos;
unsigned maxLen, offset;
UInt32 *hash;
GET_MATCHES_HEADER(4)
@@ -632,7 +688,8 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur)
{
distances[0] = maxLen = 2;
maxLen = 2;
distances[0] = 2;
distances[1] = d2 - 1;
offset = 2;
}
@@ -648,10 +705,10 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
if (offset != 0)
{
UPDATE_maxLen
distances[(size_t)offset - 2] = maxLen;
distances[(size_t)offset - 2] = (UInt32)maxLen;
if (maxLen == lenLimit)
{
SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p));
SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p));
MOVE_POS_RET;
}
}
@@ -741,7 +798,8 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
{
UInt32 h2, h3, d2, d3, maxLen, offset, pos;
UInt32 h2, h3, d2, d3, pos;
unsigned maxLen, offset;
UInt32 *hash;
GET_MATCHES_HEADER(4)
@@ -752,7 +810,6 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
d2 = pos - hash [h2];
d3 = pos - (hash + kFix3HashSize)[h3];
curMatch = (hash + kFix4HashSize)[hv];
hash [h2] = pos;
@@ -764,7 +821,8 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur)
{
distances[0] = maxLen = 2;
maxLen = 2;
distances[0] = 2;
distances[1] = d2 - 1;
offset = 2;
}
@@ -780,7 +838,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
if (offset != 0)
{
UPDATE_maxLen
distances[(size_t)offset - 2] = maxLen;
distances[(size_t)offset - 2] = (UInt32)maxLen;
if (maxLen == lenLimit)
{
p->son[p->cyclicBufferPos] = curMatch;
@@ -791,7 +849,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
if (maxLen < 3)
maxLen = 3;
offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p),
offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p),
distances + offset, maxLen) - (distances));
MOVE_POS_RET
}
@@ -877,12 +935,12 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances)
{
UInt32 offset;
unsigned offset;
GET_MATCHES_HEADER(3)
HASH_ZIP_CALC;
curMatch = p->hash[hv];
p->hash[hv] = p->pos;
offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p),
offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p),
distances, 2) - (distances));
MOVE_POS_RET
}

95
C/LzFindMt.c
View File

@@ -1,5 +1,5 @@
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms
2017-06-10 : Igor Pavlov : Public domain */
2018-12-29 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -232,38 +232,57 @@ static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p)
#define kEmptyHashValue 0
/* #define MFMT_GM_INLINE */
#define MFMT_GM_INLINE
#ifdef MFMT_GM_INLINE
#define NO_INLINE MY_FAST_CALL
/*
we use size_t for _cyclicBufferPos instead of UInt32
to eliminate "movsx" BUG in old MSVC x64 compiler.
*/
static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son,
UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue,
UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes)
MY_NO_INLINE
static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son,
size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue,
UInt32 *distances, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes)
{
do
{
UInt32 *distances = _distances + 1;
UInt32 curMatch = pos - *hash++;
UInt32 *_distances = ++distances;
UInt32 delta = *hash++;
CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1;
CLzRef *ptr1 = son + (_cyclicBufferPos << 1);
UInt32 len0 = 0, len1 = 0;
CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1;
CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1);
unsigned len0 = 0, len1 = 0;
UInt32 cutValue = _cutValue;
UInt32 maxLen = _maxLen;
for (;;)
unsigned maxLen = (unsigned)_maxLen;
/*
if (size > 1)
{
UInt32 delta = pos - curMatch;
if (cutValue-- == 0 || delta >= _cyclicBufferSize)
UInt32 delta = *hash;
if (delta < _cyclicBufferSize)
{
UInt32 cyc1 = _cyclicBufferPos + 1;
CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1);
Byte b = *(cur + 1 - delta);
_distances[0] = pair[0];
_distances[1] = b;
}
}
*/
if (cutValue == 0 || delta >= _cyclicBufferSize)
{
*ptr0 = *ptr1 = kEmptyHashValue;
break;
}
else
for(;;)
{
CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1);
{
CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1);
const Byte *pb = cur - delta;
UInt32 len = (len0 < len1 ? len0 : len1);
unsigned len = (len0 < len1 ? len0 : len1);
UInt32 pair0 = *pair;
if (pb[len] == cur[len])
{
if (++len != lenLimit && pb[len] == cur[len])
@@ -272,54 +291,66 @@ static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *
break;
if (maxLen < len)
{
*distances++ = maxLen = len;
maxLen = len;
*distances++ = (UInt32)len;
*distances++ = delta - 1;
if (len == lenLimit)
{
*ptr1 = pair[0];
*ptr0 = pair[1];
UInt32 pair1 = pair[1];
*ptr1 = pair0;
*ptr0 = pair1;
break;
}
}
}
{
UInt32 curMatch = pos - delta;
// delta = pos - *pair;
// delta = pos - pair[((UInt32)pb[len] - (UInt32)cur[len]) >> 31];
if (pb[len] < cur[len])
{
delta = pos - pair[1];
*ptr1 = curMatch;
ptr1 = pair + 1;
curMatch = *ptr1;
len1 = len;
}
else
{
delta = pos - *pair;
*ptr0 = curMatch;
ptr0 = pair;
curMatch = *ptr0;
len0 = len;
}
}
}
if (--cutValue == 0 || delta >= _cyclicBufferSize)
{
*ptr0 = *ptr1 = kEmptyHashValue;
break;
}
}
pos++;
_cyclicBufferPos++;
cur++;
{
UInt32 num = (UInt32)(distances - _distances);
*_distances = num - 1;
_distances += num;
limit -= num;
_distances[-1] = num;
}
}
while (limit > 0 && --size != 0);
while (distances < limit && --size != 0);
*posRes = pos;
return limit;
return distances;
}
#endif
static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances)
{
UInt32 numProcessed = 0;
UInt32 curPos = 2;
UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2);
UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2
distances[1] = p->hashNumAvail;
@@ -369,8 +400,10 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances)
#else
{
UInt32 posRes;
curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue,
distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes);
curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue,
distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos,
distances + limit,
size, &posRes) - distances);
p->hashBufPos += posRes - pos;
cyclicBufferPos += posRes - pos;
p->buffer += posRes - pos;

14
C/LzFindMt.h
View File

@@ -1,5 +1,5 @@
/* LzFindMt.h -- multithreaded Match finder for LZ algorithms
2017-04-03 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#ifndef __LZ_FIND_MT_H
#define __LZ_FIND_MT_H
@@ -19,10 +19,10 @@ EXTERN_C_BEGIN
typedef struct _CMtSync
{
Bool wasCreated;
Bool needStart;
Bool exit;
Bool stopWriting;
BoolInt wasCreated;
BoolInt needStart;
BoolInt exit;
BoolInt stopWriting;
CThread thread;
CAutoResetEvent canStart;
@@ -30,8 +30,8 @@ typedef struct _CMtSync
CAutoResetEvent wasStopped;
CSemaphore freeSemaphore;
CSemaphore filledSemaphore;
Bool csWasInitialized;
Bool csWasEntered;
BoolInt csWasInitialized;
BoolInt csWasEntered;
CCriticalSection cs;
UInt32 numProcessedBlocks;
} CMtSync;

26
C/Lzma2Dec.c
View File

@@ -1,5 +1,5 @@
/* Lzma2Dec.c -- LZMA2 Decoder
2018-02-19 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
/* #define SHOW_DEBUG_INFO */
@@ -169,7 +169,7 @@ static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT s
p->processedPos += (UInt32)size;
}
void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState);
void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState);
SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
@@ -232,7 +232,7 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
if (p->state == LZMA2_STATE_DATA)
{
Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC);
BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC);
LzmaDec_InitDicAndState(&p->decoder, initDic, False);
}
@@ -254,8 +254,8 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit,
if (p->state == LZMA2_STATE_DATA)
{
Bool initDic = (p->control >= 0xE0);
Bool initState = (p->control >= 0xA0);
BoolInt initDic = (p->control >= 0xE0);
BoolInt initState = (p->control >= 0xA0);
LzmaDec_InitDicAndState(&p->decoder, initDic, initState);
p->state = LZMA2_STATE_DATA_CONT;
}
@@ -314,15 +314,15 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
while (p->state != LZMA2_STATE_ERROR)
{
if (p->state == LZMA2_STATE_FINISHED)
return LZMA_STATUS_FINISHED_WITH_MARK;
return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK;
if (outSize == 0 && !checkFinishBlock)
return LZMA_STATUS_NOT_FINISHED;
return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED;
if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT)
{
if (*srcLen == inSize)
return LZMA_STATUS_NEEDS_MORE_INPUT;
return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT;
(*srcLen)++;
p->state = Lzma2Dec_UpdateState(p, *src++);
@@ -344,7 +344,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
// checkFinishBlock is true. So we expect that block must be finished,
// We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here
// break;
return LZMA_STATUS_NOT_FINISHED;
return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED;
}
if (p->state == LZMA2_STATE_DATA)
@@ -354,7 +354,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
}
if (outSize == 0)
return LZMA_STATUS_NOT_FINISHED;
return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED;
{
SizeT inCur = inSize - *srcLen;
@@ -362,7 +362,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
if (LZMA2_IS_UNCOMPRESSED_STATE(p))
{
if (inCur == 0)
return LZMA_STATUS_NEEDS_MORE_INPUT;
return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT;
if (inCur > p->unpackSize)
inCur = p->unpackSize;
if (inCur > outSize)
@@ -381,7 +381,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
if (inCur == 0)
{
if (p->packSize != 0)
return LZMA_STATUS_NEEDS_MORE_INPUT;
return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT;
}
else if (p->state == LZMA2_STATE_DATA)
{
@@ -418,7 +418,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p,
}
p->state = LZMA2_STATE_ERROR;
return LZMA_STATUS_NOT_SPECIFIED;
return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED;
}

32
C/Lzma2DecMt.c
View File

@@ -1,5 +1,5 @@
/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread
2018-03-02 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -94,13 +94,13 @@ typedef struct
ISeqOutStream *outStream;
ICompressProgress *progress;
Bool finishMode;
Bool outSize_Defined;
BoolInt finishMode;
BoolInt outSize_Defined;
UInt64 outSize;
UInt64 outProcessed;
UInt64 inProcessed;
Bool readWasFinished;
BoolInt readWasFinished;
SRes readRes;
Byte *inBuf;
@@ -113,7 +113,7 @@ typedef struct
#ifndef _7ZIP_ST
UInt64 outProcessed_Parse;
Bool mtc_WasConstructed;
BoolInt mtc_WasConstructed;
CMtDec mtc;
CLzma2DecMtThread coders[MTDEC__THREADS_MAX];
#endif
@@ -265,7 +265,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa
t->outPreSize = 0;
// t->blockWasFinished = False;
// t->finishedWithMark = False;
t->parseStatus = LZMA_STATUS_NOT_SPECIFIED;
t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED;
t->state = MTDEC_PARSE_CONTINUE;
t->inCodeSize = 0;
@@ -277,7 +277,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa
{
ELzma2ParseStatus status;
Bool overflow;
BoolInt overflow;
UInt32 unpackRem = 0;
int checkFinishBlock = True;
@@ -477,7 +477,7 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex,
{
ELzmaStatus status;
size_t srcProcessed = srcSize;
Bool blockWasFinished =
BoolInt blockWasFinished =
((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK
|| t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK);
@@ -526,15 +526,15 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex,
#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24)
static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex,
Bool needWriteToStream,
BoolInt needWriteToStream,
const Byte *src, size_t srcSize,
Bool *needContinue, Bool *canRecode)
BoolInt *needContinue, BoolInt *canRecode)
{
CLzma2DecMt *me = (CLzma2DecMt *)pp;
const CLzma2DecMtThread *t = &me->coders[coderIndex];
size_t size = t->outCodeSize;
const Byte *data = t->outBuf;
Bool needContinue2 = True;
BoolInt needContinue2 = True;
PRF_STR_INT_2("Write", coderIndex, srcSize);
@@ -633,7 +633,7 @@ static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p)
static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p
#ifndef _7ZIP_ST
, Bool tMode
, BoolInt tMode
#endif
)
{
@@ -674,8 +674,8 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p
SRes res;
SizeT outProcessed;
Bool outFinished;
Bool needStop;
BoolInt outFinished;
BoolInt needStop;
if (inPos == inLim)
{
@@ -810,7 +810,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp,
{
CLzma2DecMt *p = (CLzma2DecMt *)pp;
#ifndef _7ZIP_ST
Bool tMode;
BoolInt tMode;
#endif
*inProcessed = 0;
@@ -903,7 +903,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp,
vt.Write = Lzma2DecMt_MtCallback_Write;
{
Bool needContinue = False;
BoolInt needContinue = False;
SRes res = MtDec_Code(&p->mtc);

8
C/Lzma2Enc.c
View File

@@ -1,5 +1,5 @@
/* Lzma2Enc.c -- LZMA2 Encoder
2018-04-27 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -115,7 +115,7 @@ SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 k
ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen,
UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig);
SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit,
SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit,
Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize);
const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp);
void LzmaEnc_Finish(CLzmaEncHandle pp);
@@ -133,7 +133,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf,
size_t packSize = packSizeLimit;
UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX;
unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0);
Bool useCopyBlock;
BoolInt useCopyBlock;
SRes res;
*packSizeRes = 0;
@@ -373,7 +373,7 @@ typedef struct
size_t outBufSize; /* size of allocated outBufs[i] */
size_t outBufsDataSizes[MTCODER__BLOCKS_MAX];
Bool mtCoder_WasConstructed;
BoolInt mtCoder_WasConstructed;
CMtCoder mtCoder;
Byte *outBufs[MTCODER__BLOCKS_MAX];

8
C/Lzma86Enc.c
View File

@@ -1,5 +1,5 @@
/* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder
2016-05-16 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -18,7 +18,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen,
{
size_t outSize2 = *destLen;
Byte *filteredStream;
Bool useFilter;
BoolInt useFilter;
int mainResult = SZ_ERROR_OUTPUT_EOF;
CLzmaEncProps props;
LzmaEncProps_Init(&props);
@@ -56,7 +56,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen,
{
size_t minSize = 0;
Bool bestIsFiltered = False;
BoolInt bestIsFiltered = False;
/* passes for SZ_FILTER_AUTO:
0 - BCJ + LZMA
@@ -71,7 +71,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen,
size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE;
size_t outPropsSize = 5;
SRes curRes;
Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1);
BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1);
if (curModeIsFiltered && !bestIsFiltered)
break;
if (useFilter && i == 0)

32
C/LzmaDec.c
View File

@@ -1,13 +1,13 @@
/* LzmaDec.c -- LZMA Decoder
2018-02-28 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
#include <string.h>
/* #include "CpuArch.h" */
#include "LzmaDec.h"
#include <string.h>
#define kNumTopBits 24
#define kTopValue ((UInt32)1 << kNumTopBits)
@@ -19,7 +19,7 @@
#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); }
#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound)
#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound)
#define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits));
#define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits));
#define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \
@@ -66,7 +66,7 @@
#define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); }
#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound)
#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound)
#define UPDATE_0_CHECK range = bound;
#define UPDATE_1_CHECK range -= bound; code -= bound;
#define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \
@@ -539,7 +539,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit
curLen = ((rem < len) ? (unsigned)rem : len);
pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0);
processedPos += curLen;
processedPos += (UInt32)curLen;
len -= curLen;
if (curLen <= dicBufSize - pos)
@@ -547,7 +547,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit
Byte *dest = dic + dicPos;
ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos;
const Byte *lim = dest + curLen;
dicPos += curLen;
dicPos += (SizeT)curLen;
do
*(dest) = (Byte)*(dest + src);
while (++dest != lim);
@@ -572,14 +572,14 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit
p->buf = buf;
p->range = range;
p->code = code;
p->remainLen = len;
p->remainLen = (UInt32)len;
p->dicPos = dicPos;
p->processedPos = processedPos;
p->reps[0] = rep0;
p->reps[1] = rep1;
p->reps[2] = rep2;
p->reps[3] = rep3;
p->state = state;
p->state = (UInt32)state;
return SZ_OK;
}
@@ -601,8 +601,8 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit)
if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len)
p->checkDicSize = p->prop.dicSize;
p->processedPos += len;
p->remainLen -= len;
p->processedPos += (UInt32)len;
p->remainLen -= (UInt32)len;
while (len != 0)
{
len--;
@@ -850,7 +850,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS
}
void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState)
void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState)
{
p->remainLen = kMatchSpecLenStart + 1;
p->tempBufSize = 0;
@@ -979,10 +979,10 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr
p->tempBufSize = rem;
if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow)
{
int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem);
int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem);
if (dummyRes == DUMMY_ERROR)
{
(*srcLen) += lookAhead;
(*srcLen) += (SizeT)lookAhead;
*status = LZMA_STATUS_NEEDS_MORE_INPUT;
return SZ_OK;
}
@@ -1005,9 +1005,9 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr
return SZ_ERROR_FAIL; /* some internal error */
lookAhead -= rem;
}
(*srcLen) += lookAhead;
(*srcLen) += (SizeT)lookAhead;
src += lookAhead;
inSize -= lookAhead;
inSize -= (SizeT)lookAhead;
p->tempBufSize = 0;
}
}

709
C/LzmaEnc.c
View File

File diff suppressed because it is too large Load Diff

8
C/MtCoder.c
View File

@@ -1,5 +1,5 @@
/* MtCoder.c -- Multi-thread Coder
2018-02-21 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -119,7 +119,7 @@ static SRes ThreadFunc2(CMtCoderThread *t)
unsigned bi;
SRes res;
SRes res2;
Bool finished;
BoolInt finished;
unsigned bufIndex;
size_t size;
const Byte *inData;
@@ -294,7 +294,7 @@ static SRes ThreadFunc2(CMtCoderThread *t)
if (++wi >= mtc->numBlocksMax)
wi = 0;
{
Bool isReady;
BoolInt isReady;
CriticalSection_Enter(&mtc->cs);
@@ -547,7 +547,7 @@ SRes MtCoder_Code(CMtCoder *p)
{
const CMtCoderBlock *block = &p->blocks[bi];
unsigned bufIndex = block->bufIndex;
Bool finished = block->finished;
BoolInt finished = block->finished;
if (res == SZ_OK && block->res != SZ_OK)
res = block->res;

6
C/MtCoder.h
View File

@@ -1,5 +1,5 @@
/* MtCoder.h -- Multi-thread Coder
2018-02-21 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#ifndef __MT_CODER_H
#define __MT_CODER_H
@@ -67,7 +67,7 @@ typedef struct
{
SRes res;
unsigned bufIndex;
Bool finished;
BoolInt finished;
} CMtCoderBlock;
@@ -97,7 +97,7 @@ typedef struct _CMtCoder
CAutoResetEvent readEvent;
CSemaphore blocksSemaphore;
Bool stopReading;
BoolInt stopReading;
SRes readRes;
#ifdef MTCODER__USE_WRITE_THREAD

29
C/MtDec.c
View File

@@ -1,5 +1,5 @@
/* MtDec.c -- Multi-thread Decoder
2018-03-02 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -88,12 +88,13 @@ static WRes ArEvent_OptCreate_And_Reset(CEvent *p)
}
typedef struct
struct __CMtDecBufLink
{
void *next;
struct __CMtDecBufLink *next;
void *pad[3];
} CMtDecBufLink;
};
typedef struct __CMtDecBufLink CMtDecBufLink;
#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink)
#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET)
@@ -197,7 +198,7 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize)
}
static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, Bool *wasInterrupted)
static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted)
{
SRes res;
CriticalSection_Enter(&p->mtProgress.cs);
@@ -207,7 +208,7 @@ static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, Bool *wasInter
return res;
}
static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, Bool *wasInterrupted)
static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, BoolInt *wasInterrupted)
{
SRes res;
CriticalSection_Enter(&p->mtProgress.cs);
@@ -271,9 +272,9 @@ static WRes ThreadFunc2(CMtDecThread *t)
for (;;)
{
SRes res, codeRes;
Bool wasInterrupted, isAllocError, overflow, finish;
BoolInt wasInterrupted, isAllocError, overflow, finish;
SRes threadingErrorSRes;
Bool needCode, needWrite, needContinue;
BoolInt needCode, needWrite, needContinue;
size_t inDataSize_Start;
UInt64 inDataSize;
@@ -289,7 +290,7 @@ static WRes ThreadFunc2(CMtDecThread *t)
Byte *afterEndData = NULL;
size_t afterEndData_Size = 0;
Bool canCreateNewThread = False;
BoolInt canCreateNewThread = False;
// CMtDecCallbackInfo parse;
CMtDecThread *nextThread;
@@ -629,7 +630,7 @@ static WRes ThreadFunc2(CMtDecThread *t)
if (res == SZ_OK && needCode && codeRes == SZ_OK)
{
Bool isStartBlock = True;
BoolInt isStartBlock = True;
CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf;
for (;;)
@@ -691,9 +692,9 @@ static WRes ThreadFunc2(CMtDecThread *t)
RINOK_THREAD(Event_Wait(&t->canWrite));
{
Bool isErrorMode = False;
Bool canRecode = True;
Bool needWriteToStream = needWrite;
BoolInt isErrorMode = False;
BoolInt canRecode = True;
BoolInt needWriteToStream = needWrite;
if (p->exitThread) return 0; // it's never executed in normal cases

24
C/MtDec.h
View File

@@ -1,5 +1,5 @@
/* MtDec.h -- Multi-thread Decoder
2018-03-02 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#ifndef __MT_DEC_H
#define __MT_DEC_H
@@ -76,7 +76,7 @@ typedef struct
// out
EMtDecParseState state;
Bool canCreateNewThread;
BoolInt canCreateNewThread;
UInt64 outPos; // check it (size_t)
} CMtDecCallbackInfo;
@@ -107,11 +107,11 @@ typedef struct
if (*canRecode), we didn't flush current block data, so we still can decode current block later.
*/
SRes (*Write)(void *p, unsigned coderIndex,
Bool needWriteToStream,
BoolInt needWriteToStream,
const Byte *src, size_t srcSize,
// int srcFinished,
Bool *needContinue,
Bool *canRecode);
BoolInt *needContinue,
BoolInt *canRecode);
} IMtDecCallback;
@@ -140,22 +140,22 @@ typedef struct _CMtDec
size_t allocatedBufsSize;
Bool exitThread;
BoolInt exitThread;
WRes exitThreadWRes;
UInt64 blockIndex;
Bool isAllocError;
Bool overflow;
BoolInt isAllocError;
BoolInt overflow;
SRes threadingErrorSRes;
Bool needContinue;
BoolInt needContinue;
// CAutoResetEvent finishedEvent;
SRes readRes;
SRes codeRes;
Bool wasInterrupted;
BoolInt wasInterrupted;
unsigned numStartedThreads_Limit;
unsigned numStartedThreads;
@@ -164,14 +164,14 @@ typedef struct _CMtDec
size_t crossStart;
size_t crossEnd;
UInt64 readProcessed;
Bool readWasFinished;
BoolInt readWasFinished;
UInt64 inProcessed;
unsigned filledThreadStart;
unsigned numFilledThreads;
#ifndef _7ZIP_ST
Bool needInterrupt;
BoolInt needInterrupt;
UInt64 interruptIndex;
CMtProgress mtProgress;
CMtDecThread threads[MTDEC__THREADS_MAX];

6
C/Ppmd7.c
View File

@@ -1,5 +1,5 @@
/* Ppmd7.c -- PPMdH codec
2017-04-03 : Igor Pavlov : Public domain
2018-07-04 : Igor Pavlov : Public domain
This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */
#include "Precomp.h"
@@ -95,7 +95,7 @@ void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc)
p->Base = 0;
}
Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc)
BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc)
{
if (!p->Base || p->Size != size)
{
@@ -342,7 +342,7 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder)
p->DummySee.Count = 64; /* unused */
}
static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip)
static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip)
{
CPpmd_State upState;
CTX_PTR c = p->MinContext;

6
C/Ppmd7.h
View File

@@ -1,5 +1,5 @@
/* Ppmd7.h -- PPMdH compression codec
2017-04-03 : Igor Pavlov : Public domain
2018-07-04 : Igor Pavlov : Public domain
This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */
/* This code supports virtual RangeDecoder and includes the implementation
@@ -60,7 +60,7 @@ typedef struct
} CPpmd7;
void Ppmd7_Construct(CPpmd7 *p);
Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc);
BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc);
void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc);
void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder);
#define Ppmd7_WasAllocated(p) ((p)->Base != NULL)
@@ -115,7 +115,7 @@ typedef struct
} CPpmd7z_RangeDec;
void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p);
Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p);
BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p);
#define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0)
int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc);

4
C/Ppmd7Dec.c
View File

@@ -1,5 +1,5 @@
/* Ppmd7Dec.c -- PPMdH Decoder
2017-04-03 : Igor Pavlov : Public domain
2018-07-04 : Igor Pavlov : Public domain
This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */
#include "Precomp.h"
@@ -8,7 +8,7 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */
#define kTopValue (1 << 24)
Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p)
BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p)
{
unsigned i;
p->Code = 0;

6
C/Ppmd8.c
View File

@@ -1,5 +1,5 @@
/* Ppmd8.c -- PPMdI codec
2017-04-03 : Igor Pavlov : Public domain
2018-07-04 : Igor Pavlov : Public domain
This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */
#include "Precomp.h"
@@ -93,7 +93,7 @@ void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc)
p->Base = 0;
}
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc)
BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc)
{
if (!p->Base || p->Size != size)
{
@@ -605,7 +605,7 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR c1
}
}
static CTX_PTR CreateSuccessors(CPpmd8 *p, Bool skip, CPpmd_State *s1, CTX_PTR c)
static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PTR c)
{
CPpmd_State upState;
Byte flags;

6
C/Ppmd8.h
View File

@@ -1,5 +1,5 @@
/* Ppmd8.h -- PPMdI codec
2017-04-03 : Igor Pavlov : Public domain
2018-07-04 : Igor Pavlov : Public domain
This code is based on:
PPMd var.I (2002): Dmitry Shkarin : Public domain
Carryless rangecoder (1999): Dmitry Subbotin : Public domain */
@@ -86,7 +86,7 @@ typedef struct
} CPpmd8;
void Ppmd8_Construct(CPpmd8 *p);
Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc);
BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc);
void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc);
void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod);
#define Ppmd8_WasAllocated(p) ((p)->Base != NULL)
@@ -121,7 +121,7 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked, UInt32 *scale);
/* ---------- Decode ---------- */
Bool Ppmd8_RangeDec_Init(CPpmd8 *p);
BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p);
#define Ppmd8_RangeDec_IsFinishedOK(p) ((p)->Code == 0)
int Ppmd8_DecodeSymbol(CPpmd8 *p); /* returns: -1 as EndMarker, -2 as DataError */

4
C/Ppmd8Dec.c
View File

@@ -1,5 +1,5 @@
/* Ppmd8Dec.c -- PPMdI Decoder
2017-04-03 : Igor Pavlov : Public domain
2018-07-04 : Igor Pavlov : Public domain
This code is based on:
PPMd var.I (2002): Dmitry Shkarin : Public domain
Carryless rangecoder (1999): Dmitry Subbotin : Public domain */
@@ -11,7 +11,7 @@ This code is based on:
#define kTop (1 << 24)
#define kBot (1 << 15)
Bool Ppmd8_RangeDec_Init(CPpmd8 *p)
BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p)
{
unsigned i;
p->Low = 0;

14
C/Util/7z/7zMain.c
View File

@@ -1,5 +1,5 @@
/* 7zMain.c - Test application for 7z Decoder
2018-04-19 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -176,7 +176,7 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s
char defaultChar = '_';
BOOL defUsed;
unsigned numChars = 0;
numChars = WideCharToMultiByte(codePage, 0, s, len, (char *)buf->data, size, &defaultChar, &defUsed);
numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed);
if (numChars == 0 || numChars >= size)
return SZ_ERROR_FAIL;
buf->data[numChars] = 0;
@@ -202,7 +202,7 @@ static WRes MyCreateDir(const UInt16 *name)
{
#ifdef USE_WINDOWS_FILE
return CreateDirectoryW(name, NULL) ? 0 : GetLastError();
return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError();
#else
@@ -227,7 +227,7 @@ static WRes MyCreateDir(const UInt16 *name)
static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name)
{
#ifdef USE_WINDOWS_FILE
return OutFile_OpenW(p, name);
return OutFile_OpenW(p, (LPCWSTR)name);
#else
CBuf buf;
WRes res;
@@ -354,7 +354,7 @@ static void PrintError(char *s)
PrintLF();
}
static void GetAttribString(UInt32 wa, Bool isDir, char *s)
static void GetAttribString(UInt32 wa, BoolInt isDir, char *s)
{
#ifdef USE_WINDOWS_FILE
s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.');
@@ -430,7 +430,7 @@ int MY_CDECL main(int numargs, char *args[])
res = SZ_OK;
{
lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize);
lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize);
if (!lookStream.buf)
res = SZ_ERROR_MEM;
else
@@ -647,7 +647,7 @@ int MY_CDECL main(int numargs, char *args[])
We remove posix bits, if we detect posix mode field */
if ((attrib & 0xF0000000) != 0)
attrib &= 0x7FFF;
SetFileAttributesW(destPath, attrib);
SetFileAttributesW((LPCWSTR)destPath, attrib);
}
#endif
}

78
C/Util/7zipInstall/7zipInstall.c
View File

@@ -1,5 +1,5 @@
/* 7zipInstall.c - 7-Zip Installer
2017-08-28 : Igor Pavlov : Public domain */
2019-02-19 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -82,16 +82,16 @@ static LPCWSTR const k_Reg_Path32 = L"Path"
#define k_Reg_WOW_Flag_32 0
#endif
#define k_7zip_CLSID L"{23170F69-0803-278A-1000-000100020001}"
#define k_7zip_CLSID L"{23170F69-20BB-278A-1000-000100020000}"
static LPCWSTR const k_Reg_CLSID_7zip = L"CLSID\\" k_7zip_CLSID;
static LPCWSTR const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\InprocServer32";
#define g_AllUsers True
static Bool g_Install_was_Pressed;
static Bool g_Finished;
static Bool g_SilentMode;
static BoolInt g_Install_was_Pressed;
static BoolInt g_Finished;
static BoolInt g_SilentMode;
static HWND g_HWND;
static HWND g_Path_HWND;
@@ -121,13 +121,51 @@ static void PrintErrorMessage(const char *s)
}
typedef DWORD (WINAPI * Func_GetFileVersionInfoSizeW)(LPCWSTR lptstrFilename, LPDWORD lpdwHandle);
typedef BOOL (WINAPI * Func_GetFileVersionInfoW)(LPCWSTR lptstrFilename, DWORD dwHandle, DWORD dwLen, LPVOID lpData);
typedef BOOL (WINAPI * Func_VerQueryValueW)(const LPVOID pBlock, LPWSTR lpSubBlock, LPVOID * lplpBuffer, PUINT puLen);
static HMODULE g_version_dll_hModule;
static DWORD GetFileVersion(LPCWSTR s)
{
DWORD size = 0;
BYTE *vi = NULL;
void *vi = NULL;
DWORD version = 0;
size = GetFileVersionInfoSizeW(s, NULL);
Func_GetFileVersionInfoSizeW my_GetFileVersionInfoSizeW;
Func_GetFileVersionInfoW my_GetFileVersionInfoW;
Func_VerQueryValueW my_VerQueryValueW;
if (!g_version_dll_hModule)
{
wchar_t buf[MAX_PATH + 100];
{
unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2);
if (len == 0 || len > MAX_PATH)
return 0;
}
{
unsigned pos = (unsigned)lstrlenW(buf);
if (buf[pos - 1] != '\\')
buf[pos++] = '\\';
lstrcpyW(buf + pos, L"version.dll");
}
g_version_dll_hModule = LoadLibraryW(buf);
if (!g_version_dll_hModule)
return 0;
}
my_GetFileVersionInfoSizeW = (Func_GetFileVersionInfoSizeW)GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoSizeW");
my_GetFileVersionInfoW = (Func_GetFileVersionInfoW)GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoW");
my_VerQueryValueW = (Func_VerQueryValueW)GetProcAddress(g_version_dll_hModule, "VerQueryValueW");
if (!my_GetFileVersionInfoSizeW
|| !my_GetFileVersionInfoW
|| !my_VerQueryValueW)
return 0;
size = my_GetFileVersionInfoSizeW(s, NULL);
if (size == 0)
return 0;
@@ -135,11 +173,11 @@ static DWORD GetFileVersion(LPCWSTR s)
if (!vi)
return 0;
if (GetFileVersionInfoW(s, 0, size, vi))
if (my_GetFileVersionInfoW(s, 0, size, vi))
{
VS_FIXEDFILEINFO *fi = NULL;
UINT fiLen = 0;
if (VerQueryValueW(vi, L"\\", (LPVOID *)&fi, &fiLen))
if (my_VerQueryValueW(vi, L"\\", (LPVOID *)&fi, &fiLen))
version = fi->dwFileVersionMS;
}
@@ -270,7 +308,7 @@ static int MyRegistry_QueryString2(HKEY hKey, LPCWSTR keyName, LPCWSTR valName,
if (res != ERROR_SUCCESS)
return False;
{
Bool res2 = MyRegistry_QueryString(key, valName, dest);
BoolInt res2 = MyRegistry_QueryString(key, valName, dest);
RegCloseKey(key);
return res2;
}
@@ -343,7 +381,7 @@ static LONG MyRegistry_CreateKeyAndVal_32(HKEY parentKey, LPCWSTR keyName, LPCWS
#define kSignatureSearchLimit (1 << 22)
static Bool FindSignature(CSzFile *stream, UInt64 *resPos)
static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos)
{
Byte buf[kBufSize];
size_t numPrevBytes = 0;
@@ -431,7 +469,7 @@ int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM lp, LPARAM data)
return 0;
}
static Bool MyBrowseForFolder(HWND owner, LPCWSTR title, UINT ulFlags,
static BoolInt MyBrowseForFolder(HWND owner, LPCWSTR title, UINT ulFlags,
LPCWSTR initialFolder, LPWSTR resultPath)
{
WCHAR displayName[MAX_PATH];
@@ -694,7 +732,7 @@ static void SetShellProgramsGroup(HWND hwndOwner)
for (; i < 3; i++)
{
Bool isOK = True;
BoolInt isOK = True;
WCHAR link[MAX_PATH + 40];
WCHAR destPath[MAX_PATH + 40];
@@ -874,7 +912,7 @@ static void WriteShellEx()
static const wchar_t *GetCmdParam(const wchar_t *s)
{
Bool quoteMode = False;
BoolInt quoteMode = False;
for (;; s++)
{
wchar_t c = *s;
@@ -988,7 +1026,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
if (path[0] == 0)
{
HKEY key = 0;
Bool ok = False;
BoolInt ok = False;
LONG res = RegOpenKeyExW(HKEY_CURRENT_USER, k_Reg_Software_7zip, 0, KEY_READ | k_Reg_WOW_Flag, &key);
if (res == ERROR_SUCCESS)
{
@@ -1101,9 +1139,9 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
}
static Bool GetErrorMessage(DWORD errorCode, WCHAR *message)
static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message)
{
LPVOID msgBuf;
LPWSTR msgBuf;
if (FormatMessageW(
FORMAT_MESSAGE_ALLOCATE_BUFFER
| FORMAT_MESSAGE_FROM_SYSTEM
@@ -1201,7 +1239,7 @@ if (res == SZ_OK)
if (res == SZ_OK)
{
lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize);
lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize);
if (!lookStream.buf)
res = SZ_ERROR_MEM;
else
@@ -1272,7 +1310,7 @@ if (res == SZ_OK)
temp = path + pathLen;
SzArEx_GetFileNameUtf16(&db, i, temp);
SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp);
if (!g_SilentMode)
SetWindowTextW(g_InfoLine_HWND, temp);
@@ -1313,7 +1351,7 @@ if (res == SZ_OK)
}
{
// Bool skipFile = False;
// BoolInt skipFile = False;
wcscpy(origPath, path);

2
C/Util/7zipInstall/makefile
View File

@@ -1,8 +1,6 @@
PROG = 7zipInstall.exe
MY_FIXED = 1
LIBS = $(LIBS) version.lib
!IFDEF _64BIT_INSTALLER
CFLAGS = $(CFLAGS) -D_64BIT_INSTALLER
!ENDIF

42
C/Util/7zipUninstall/7zipUninstall.c
View File

@@ -1,5 +1,5 @@
/* 7zipUninstall.c - 7-Zip Uninstaller
2018-03-01 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -62,7 +62,7 @@ static const WCHAR * const k_Reg_Path32 = L"Path"
#define k_Reg_WOW_Flag_32 0
#endif
#define k_7zip_CLSID L"{23170F69-0803-278A-1000-000100020001}"
#define k_7zip_CLSID L"{23170F69-20BB-278A-1000-000100020000}"
static const WCHAR * const k_Reg_CLSID_7zip = L"CLSID\\" k_7zip_CLSID;
static const WCHAR * const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\InprocServer32";
@@ -70,9 +70,9 @@ static const WCHAR * const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\
#define g_AllUsers True
static Bool g_Install_was_Pressed;
static Bool g_Finished;
static Bool g_SilentMode;
static BoolInt g_Install_was_Pressed;
static BoolInt g_Finished;
static BoolInt g_SilentMode;
static HWND g_HWND;
static HWND g_Path_HWND;
@@ -94,7 +94,7 @@ static const WCHAR * const kUninstallExe = L"Uninstall.exe";
#define MAKE_CHAR_UPPER(c) ((((c) >= 'a' && (c) <= 'z') ? (c) -= 0x20 : (c)))
static Bool AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2)
static BoolInt AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2)
{
for (;;)
{
@@ -107,7 +107,7 @@ static Bool AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2)
}
}
static Bool IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2)
static BoolInt IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2)
{
for (;;)
{
@@ -149,7 +149,7 @@ static int MyRegistry_QueryString2(HKEY hKey, LPCWSTR keyName, LPCWSTR valName,
if (res != ERROR_SUCCESS)
return False;
{
Bool res2 = MyRegistry_QueryString(key, valName, dest);
BoolInt res2 = MyRegistry_QueryString(key, valName, dest);
RegCloseKey(key);
return res2;
}
@@ -180,7 +180,7 @@ static int MyRegistry_QueryString2_32(HKEY hKey, LPCWSTR keyName, LPCWSTR valNam
if (res != ERROR_SUCCESS)
return False;
{
Bool res2 = MyRegistry_QueryString(key, valName, dest);
BoolInt res2 = MyRegistry_QueryString(key, valName, dest);
RegCloseKey(key);
return res2;
}
@@ -283,7 +283,7 @@ static void SetShellProgramsGroup(HWND hwndOwner)
for (; i < 3; i++)
{
// Bool isOK = True;
// BoolInt isOK = True;
WCHAR link[MAX_PATH + 40];
WCHAR destPath[MAX_PATH + 40];
@@ -300,7 +300,7 @@ static void SetShellProgramsGroup(HWND hwndOwner)
{
const size_t baseLen = wcslen(link);
unsigned k;
Bool needDelete = False;
BoolInt needDelete = False;
for (k = 0; k < 2; k++)
{
@@ -347,7 +347,7 @@ static const WCHAR * const k_AppPaths_7zFm = L"Software\\Microsoft\\Windows\\Cur
static const WCHAR * const k_Uninstall_7zip = k_REG_Uninstall L"7-Zip-Zstandard";
static Bool AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name)
static BoolInt AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name)
{
if (!IsString1PrefixedByString2_NoCase(s, prefix))
return False;
@@ -444,7 +444,7 @@ static void WriteCLSID()
static const wchar_t *GetCmdParam(const wchar_t *s)
{
Bool quoteMode = False;
BoolInt quoteMode = False;
for (;; s++)
{
wchar_t c = *s;
@@ -469,7 +469,7 @@ static void RemoveQuotes(wchar_t *s)
}
}
static Bool DoesFileOrDirExist()
static BoolInt DoesFileOrDirExist()
{
return (GetFileAttributesW(path) != INVALID_FILE_ATTRIBUTES);
}
@@ -491,7 +491,7 @@ static BOOL RemoveFileAfterReboot()
#define IS_LIMIT_CHAR(c) (c == 0 || c == ' ')
static Bool IsThereSpace(const wchar_t *s)
static BoolInt IsThereSpace(const wchar_t *s)
{
for (;;)
{
@@ -505,7 +505,7 @@ static Bool IsThereSpace(const wchar_t *s)
static void AddPathParam(wchar_t *dest, const wchar_t *src)
{
Bool needQuote = IsThereSpace(src);
BoolInt needQuote = IsThereSpace(src);
if (needQuote)
wcscat(dest, L"\"");
wcscat(dest, src);
@@ -515,9 +515,9 @@ static void AddPathParam(wchar_t *dest, const wchar_t *src)
static Bool GetErrorMessage(DWORD errorCode, WCHAR *message)
static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message)
{
LPVOID msgBuf;
LPWSTR msgBuf;
if (FormatMessageW(
FORMAT_MESSAGE_ALLOCATE_BUFFER
| FORMAT_MESSAGE_FROM_SYSTEM
@@ -586,7 +586,7 @@ static int Install()
SRes res = SZ_OK;
WRes winRes = 0;
// Bool needReboot = False;
// BoolInt needReboot = False;
const size_t pathLen = wcslen(path);
if (!g_SilentMode)
@@ -805,7 +805,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
lpCmdLine, int nCmdShow)
{
const wchar_t *cmdParams;
Bool useTemp = True;
BoolInt useTemp = True;
UNUSED_VAR(hPrevInstance)
UNUSED_VAR(lpCmdLine)
@@ -1005,7 +1005,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
if (path[0] == 0)
{
HKEY key = 0;
Bool ok = False;
BoolInt ok = False;
LONG res = RegOpenKeyExW(HKEY_CURRENT_USER, k_Reg_Software_7zip, 0, KEY_READ | k_Reg_WOW_Flag, &key);
if (res == ERROR_SUCCESS)
{

4
C/Util/Lzma/LzmaUtil.c
View File

@@ -1,5 +1,5 @@
/* LzmaUtil.c -- Test application for LZMA compression
2017-04-27 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "../../Precomp.h"
@@ -177,7 +177,7 @@ static int main2(int numArgs, const char *args[], char *rs)
char c;
int res;
int encodeMode;
Bool useOutFile = False;
BoolInt useOutFile = False;
FileSeqInStream_CreateVTable(&inStream);
File_Construct(&inStream.file);

16
C/Util/SfxSetup/SfxSetup.c
View File

@@ -1,5 +1,5 @@
/* SfxSetup.c - 7z SFX Setup
2017-04-04 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -127,7 +127,7 @@ static WRes MyCreateDir(const WCHAR *name)
#define kSignatureSearchLimit (1 << 22)
static Bool FindSignature(CSzFile *stream, UInt64 *resPos)
static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos)
{
Byte buf[kBufferSize];
size_t numPrevBytes = 0;
@@ -163,7 +163,7 @@ static Bool FindSignature(CSzFile *stream, UInt64 *resPos)
}
}
static Bool DoesFileOrDirExist(const WCHAR *path)
static BoolInt DoesFileOrDirExist(const WCHAR *path)
{
WIN32_FIND_DATAW fd;
HANDLE handle;
@@ -254,7 +254,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
DWORD winRes;
const wchar_t *cmdLineParams;
const char *errorMessage = NULL;
Bool useShellExecute = True;
BoolInt useShellExecute = True;
DWORD exitCode = 0;
LoadSecurityDlls();
@@ -287,7 +287,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
cmdLineParams = GetCommandLineW();
#ifndef UNDER_CE
{
Bool quoteMode = False;
BoolInt quoteMode = False;
for (;; cmdLineParams++)
{
wchar_t c = *cmdLineParams;
@@ -379,7 +379,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
if (res == SZ_OK)
{
lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize);
lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize);
if (!lookStream.buf)
res = SZ_ERROR_MEM;
else
@@ -420,7 +420,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
temp = path + pathLen;
SzArEx_GetFileNameUtf16(&db, i, temp);
SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp);
{
res = SzArEx_Extract(&db, &lookStream.vt, i,
&blockIndex, &outBuffer, &outBufferSize,
@@ -527,7 +527,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance,
{
WCHAR *temp = path + pathLen;
UInt32 j;
SzArEx_GetFileNameUtf16(&db, executeFileIndex, temp);
SzArEx_GetFileNameUtf16(&db, executeFileIndex, (UInt16 *)temp);
for (j = 0; temp[j] != 0; j++)
if (temp[j] == '/')
temp[j] = CHAR_PATH_SEPARATOR;

20
C/Xz.h
View File

@@ -1,5 +1,5 @@
/* Xz.h - Xz interface
2018-02-28 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#ifndef __XZ_H
#define __XZ_H
@@ -53,7 +53,7 @@ typedef struct
#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0)
SRes XzBlock_Parse(CXzBlock *p, const Byte *header);
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes);
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes);
/* ---------- xz stream ---------- */
@@ -186,10 +186,10 @@ typedef struct
Byte *outBuf;
size_t outBufSize;
size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode)
Bool wasFinished;
BoolInt wasFinished;
SRes res;
ECoderStatus status;
// Bool SingleBufMode;
// BoolInt SingleBufMode;
int finished[MIXCODER_NUM_FILTERS_MAX - 1];
size_t pos[MIXCODER_NUM_FILTERS_MAX - 1];
@@ -241,9 +241,9 @@ typedef struct
CXzCheck check;
CSha256 sha;
Bool parseMode;
Bool headerParsedOk;
Bool decodeToStreamSignature;
BoolInt parseMode;
BoolInt headerParsedOk;
BoolInt decodeToStreamSignature;
unsigned decodeOnlyOneBlock;
Byte *outBuf;
@@ -335,7 +335,7 @@ SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen,
ECoderFinishMode finishMode, ECoderStatus *status);
Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p);
BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p);
/*
XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes,
@@ -365,7 +365,7 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p);
*/
void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p);
Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p);
BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p);
#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags))
@@ -378,7 +378,7 @@ typedef struct
{
size_t inBufSize_ST;
size_t outStep_ST;
Bool ignoreErrors;
BoolInt ignoreErrors;
#ifndef _7ZIP_ST
unsigned numThreads;

87
C/XzDec.c
View File

@@ -1,5 +1,5 @@
/* XzDec.c -- Xz Decode
2018-04-24 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -245,7 +245,7 @@ SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocP
CBraState *decoder;
if (id < XZ_ID_Delta || id > XZ_ID_SPARC)
return SZ_ERROR_UNSUPPORTED;
decoder = p->p;
decoder = (CBraState *)p->p;
if (!decoder)
{
decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState));
@@ -341,7 +341,7 @@ static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc)
typedef struct
{
CLzma2Dec decoder;
Bool outBufMode;
BoolInt outBufMode;
} CLzma2Dec_Spec;
@@ -400,7 +400,7 @@ static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *s
res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2);
// *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK);
// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder
*status = status2;
*status = (ECoderStatus)status2;
return res;
}
@@ -637,8 +637,8 @@ static SRes MixCoder_Code(CMixCoder *p,
for (;;)
{
Bool processed = False;
Bool allFinished = True;
BoolInt processed = False;
BoolInt allFinished = True;
SRes resMain = SZ_OK;
unsigned i;
@@ -761,7 +761,7 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf)
return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED;
}
static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf)
static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf)
{
return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2)
&& GetUi32(buf) == CrcCalc(buf + 4, 6)
@@ -775,7 +775,7 @@ static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *b
if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; }
static Bool XzBlock_AreSupportedFilters(const CXzBlock *p)
static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p)
{
unsigned numFilters = XzBlock_GetNumFilters(p) - 1;
unsigned i;
@@ -866,7 +866,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header)
static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize)
{
unsigned i;
Bool needReInit = True;
BoolInt needReInit = True;
unsigned numFilters = XzBlock_GetNumFilters(block);
if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf)))
@@ -999,8 +999,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen,
SRes res;
ECoderFinishMode finishMode2 = finishMode;
Bool srcFinished2 = srcFinished;
Bool destFinish = False;
BoolInt srcFinished2 = srcFinished;
BoolInt destFinish = False;
if (p->block.packSize != (UInt64)(Int64)-1)
{
@@ -1346,12 +1346,12 @@ SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen,
}
Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p)
BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p)
{
return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0);
}
Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p)
BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p)
{
return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0);
}
@@ -1423,18 +1423,18 @@ typedef struct
size_t outCodeSize;
ECoderStatus status;
SRes codeRes;
Bool skipMode;
// Bool finishedWithMark;
BoolInt skipMode;
// BoolInt finishedWithMark;
EMtDecParseState parseState;
Bool parsing_Truncated;
Bool atBlockHeader;
BoolInt parsing_Truncated;
BoolInt atBlockHeader;
CXzStreamFlags streamFlags;
// UInt64 numFinishedStreams
UInt64 numStreams;
UInt64 numTotalBlocks;
UInt64 numBlocks;
Bool dec_created;
BoolInt dec_created;
CXzUnpacker dec;
Byte mtPad[1 << 7];
@@ -1458,14 +1458,14 @@ typedef struct
ICompressProgress *progress;
// CXzStatInfo *stat;
Bool finishMode;
Bool outSize_Defined;
BoolInt finishMode;
BoolInt outSize_Defined;
UInt64 outSize;
UInt64 outProcessed;
UInt64 inProcessed;
UInt64 readProcessed;
Bool readWasFinished;
BoolInt readWasFinished;
SRes readRes;
SRes writeRes;
@@ -1473,14 +1473,14 @@ typedef struct
size_t outBufSize;
Byte *inBuf;
size_t inBufSize;
Bool dec_created;
CXzUnpacker dec;
ECoderStatus status;
SRes codeRes;
#ifndef _7ZIP_ST
Bool mainDecoderWasCalled;
BoolInt mainDecoderWasCalled;
// int statErrorDefined;
int finishedDecoderIndex;
@@ -1494,12 +1494,12 @@ typedef struct
// UInt64 numBadBlocks;
SRes mainErrorCode;
Bool isBlockHeaderState_Parse;
Bool isBlockHeaderState_Write;
BoolInt isBlockHeaderState_Parse;
BoolInt isBlockHeaderState_Write;
UInt64 outProcessed_Parse;
Bool parsing_Truncated;
BoolInt parsing_Truncated;
Bool mtc_WasConstructed;
BoolInt mtc_WasConstructed;
CMtDec mtc;
CXzDecMtThread coders[MTDEC__THREADS_MAX];
#endif
@@ -1525,7 +1525,8 @@ CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid)
p->outBufSize = 0;
p->inBuf = NULL;
p->inBufSize = 0;
p->dec_created = False;
XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt);
p->unpackBlockMaxSize = 0;
@@ -1572,12 +1573,8 @@ static void XzDecMt_FreeOutBufs(CXzDecMt *p)
static void XzDecMt_FreeSt(CXzDecMt *p)
{
if (p->dec_created)
{
XzUnpacker_Free(&p->dec);
p->dec_created = False;
}
if (p->outBuf)
{
@@ -1968,11 +1965,11 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex,
#define XZDECMT_STREAM_WRITE_STEP (1 << 24)
static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex,
Bool needWriteToStream,
BoolInt needWriteToStream,
const Byte *src, size_t srcSize,
// int srcFinished,
Bool *needContinue,
Bool *canRecode)
BoolInt *needContinue,
BoolInt *canRecode)
{
CXzDecMt *me = (CXzDecMt *)pp;
const CXzDecMtThread *coder = &me->coders[coderIndex];
@@ -2302,7 +2299,7 @@ void XzStatInfo_Clear(CXzStatInfo *p)
static SRes XzDecMt_Decode_ST(CXzDecMt *p
#ifndef _7ZIP_ST
, Bool tMode
, BoolInt tMode
#endif
, CXzStatInfo *stat)
{
@@ -2358,7 +2355,7 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p
for (;;)
{
SizeT outSize;
Bool finished;
BoolInt finished;
ECoderFinishMode finishMode;
SizeT inProcessed;
ECoderStatus status;
@@ -2466,7 +2463,7 @@ static SRes XzStatInfo_SetStat(const CXzUnpacker *dec,
int finishMode,
UInt64 readProcessed, UInt64 inProcessed,
SRes res, ECoderStatus status,
Bool decodingTruncated,
BoolInt decodingTruncated,
CXzStatInfo *stat)
{
UInt64 extraSize;
@@ -2531,7 +2528,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
{
CXzDecMt *p = (CXzDecMt *)pp;
#ifndef _7ZIP_ST
Bool tMode;
BoolInt tMode;
#endif
XzStatInfo_Clear(stat);
@@ -2564,14 +2561,8 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
p->codeRes = 0;
p->status = CODER_STATUS_NOT_SPECIFIED;
if (!p->dec_created)
{
XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt);
p->dec_created = True;
}
XzUnpacker_Init(&p->dec);
*isMT = False;
/*
@@ -2600,6 +2591,8 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
{
IMtDecCallback vt;
// we just free ST buffers here
// but we still keep state variables, that was set in XzUnpacker_Init()
XzDecMt_FreeSt(p);
p->outProcessed_Parse = 0;
@@ -2636,7 +2629,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
vt.Write = XzDecMt_Callback_Write;
{
Bool needContinue;
BoolInt needContinue;
SRes res = MtDec_Code(&p->mtc);
@@ -2665,7 +2658,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp,
if (!needContinue)
{
SRes codeRes;
Bool truncated = False;
BoolInt truncated = False;
ECoderStatus status;
CXzUnpacker *dec;

8
C/XzEnc.c
View File

@@ -1,5 +1,5 @@
/* XzEnc.c -- Xz Encode
2018-04-28 : Igor Pavlov : Public domain */
2019-02-02 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -366,7 +366,7 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size)
SRes res;
*size = sizeOriginal;
res = p->StateCoder.Code2(p->StateCoder.p,
data, size,
(Byte *)data, size,
p->buf + p->curPos, &srcLen,
p->srcWasFinished, CODER_FINISH_ANY,
&status);
@@ -814,7 +814,7 @@ static SRes Xz_CompressBlock(
SRes res;
Byte *outBuf = NULL;
size_t outSize = 0;
Bool useStream = (fp || inStream);
BoolInt useStream = (fp || inStream);
// useStream = True;
if (!useStream)
@@ -940,7 +940,7 @@ typedef struct
#ifndef _7ZIP_ST
unsigned checkType;
ISeqOutStream *outStream;
Bool mtCoder_WasConstructed;
BoolInt mtCoder_WasConstructed;
CMtCoder mtCoder;
CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX];
#endif

4
C/XzIn.c
View File

@@ -1,5 +1,5 @@
/* XzIn.c - Xz input
2018-02-02 : Igor Pavlov : Public domain */
2018-07-04 : Igor Pavlov : Public domain */
#include "Precomp.h"
@@ -28,7 +28,7 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream)
{ unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \
if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; }
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes)
SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes)
{
Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
unsigned headerSize;

5
C/brotli/Brotli-Adjust.sh
View File

@@ -36,10 +36,13 @@ sed -i 's|include "../common|include "./common/|g' *.c
for f in *.c; do mv $f ../br_$f; done
exit
# and also disable these msc warnings:
# then put these to "port.h"
/* disable some warnings /TR */
#pragma warning(disable : 4100)
#pragma warning(disable : 4127)
#pragma warning(disable : 4189)
#pragma warning(disable : 4295)
#pragma warning(disable : 4389)
#pragma warning(disable : 4701)

105
C/brotli/br_backward_references_hq.c
View File

@@ -330,7 +330,7 @@ static size_t ComputeMinimumCopyLength(const float start_cost,
REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */
static uint32_t ComputeDistanceShortcut(const size_t block_start,
const size_t pos,
const size_t max_backward,
const size_t max_backward_limit,
const size_t gap,
const ZopfliNode* nodes) {
const size_t clen = ZopfliNodeCopyLength(&nodes[pos]);
@@ -338,13 +338,13 @@ static uint32_t ComputeDistanceShortcut(const size_t block_start,
const size_t dist = ZopfliNodeCopyDistance(&nodes[pos]);
/* Since |block_start + pos| is the end position of the command, the copy part
starts from |block_start + pos - clen|. Distances that are greater than
this or greater than |max_backward| are static dictionary references, and
do not update the last distances. Also distance code 0 (last distance)
does not update the last distances. */
this or greater than |max_backward_limit| + |gap| are static dictionary
references, and do not update the last distances.
Also distance code 0 (last distance) does not update the last distances. */
if (pos == 0) {
return 0;
} else if (dist + clen <= block_start + pos + gap &&
dist <= max_backward + gap &&
dist <= max_backward_limit + gap &&
ZopfliNodeDistanceCode(&nodes[pos]) > 0) {
return (uint32_t)pos;
} else {
@@ -454,9 +454,11 @@ static size_t UpdateNodes(
break;
}
if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) {
/* Word dictionary -> ignore. */
continue;
}
if (backward <= max_distance) {
/* Regular backward reference. */
if (prev_ix >= cur_ix) {
continue;
}
@@ -564,14 +566,10 @@ static size_t ComputeShortestPathFromNodes(size_t num_bytes,
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
void BrotliZopfliCreateCommands(const size_t num_bytes,
const size_t block_start,
const size_t max_backward_limit,
const ZopfliNode* nodes,
int* dist_cache,
size_t* last_insert_len,
const BrotliEncoderParams* params,
Command* commands,
size_t* num_literals) {
const size_t block_start, const ZopfliNode* nodes, int* dist_cache,
size_t* last_insert_len, const BrotliEncoderParams* params,
Command* commands, size_t* num_literals) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
size_t pos = 0;
uint32_t offset = nodes[0].u.next;
size_t i;
@@ -610,18 +608,12 @@ void BrotliZopfliCreateCommands(const size_t num_bytes,
*last_insert_len += num_bytes - pos;
}
static size_t ZopfliIterate(size_t num_bytes,
size_t position,
const uint8_t* ringbuffer,
size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const size_t max_backward_limit,
const size_t gap,
const int* dist_cache,
const ZopfliCostModel* model,
const uint32_t* num_matches,
const BackwardMatch* matches,
ZopfliNode* nodes) {
static size_t ZopfliIterate(size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, const size_t gap, const int* dist_cache,
const ZopfliCostModel* model, const uint32_t* num_matches,
const BackwardMatch* matches, ZopfliNode* nodes) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
const size_t max_zopfli_len = MaxZopfliLen(params);
StartPosQueue queue;
size_t cur_match_pos = 0;
@@ -645,8 +637,8 @@ static size_t ZopfliIterate(size_t num_bytes,
while (skip) {
i++;
if (i + 3 >= num_bytes) break;
EvaluateNode(position, i, max_backward_limit, gap, dist_cache, model,
&queue, nodes);
EvaluateNode(position, i, max_backward_limit, gap,
dist_cache, model, &queue, nodes);
cur_match_pos += num_matches[i];
skip--;
}
@@ -656,11 +648,11 @@ static size_t ZopfliIterate(size_t num_bytes,
}
/* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */
size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
const size_t max_backward_limit, const int* dist_cache, HasherHandle hasher,
ZopfliNode* nodes) {
size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
const size_t max_zopfli_len = MaxZopfliLen(params);
ZopfliCostModel model;
StartPosQueue queue;
@@ -681,9 +673,11 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
const size_t pos = position + i;
const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit);
size_t skip;
size_t num_matches = FindAllMatchesH10(hasher, &params->dictionary,
ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance, gap,
params, &matches[lz_matches_offset]);
size_t num_matches;
num_matches = FindAllMatchesH10(hasher,
&params->dictionary,
ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance,
gap, params, &matches[lz_matches_offset]);
if (num_matches > 0 &&
BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) {
matches[0] = matches[num_matches - 1];
@@ -704,8 +698,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
while (skip) {
i++;
if (i + HashTypeLengthH10() - 1 >= num_bytes) break;
EvaluateNode(position, i, max_backward_limit, gap, dist_cache, &model,
&queue, nodes);
EvaluateNode(position, i, max_backward_limit, gap,
dist_cache, &model, &queue, nodes);
skip--;
}
}
@@ -714,28 +708,27 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
return ComputeShortestPathFromNodes(num_bytes, nodes);
}
void BrotliCreateZopfliBackwardReferences(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params,
HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
ZopfliNode* nodes;
nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1);
if (BROTLI_IS_OOM(m)) return;
BrotliInitZopfliNodes(nodes, num_bytes + 1);
*num_commands += BrotliZopfliComputeShortestPath(m,
num_bytes, position, ringbuffer, ringbuffer_mask,
params, max_backward_limit, dist_cache, hasher, nodes);
*num_commands += BrotliZopfliComputeShortestPath(m, num_bytes,
position, ringbuffer, ringbuffer_mask, params,
dist_cache, hasher, nodes);
if (BROTLI_IS_OOM(m)) return;
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes,
dist_cache, last_insert_len, params, commands, num_literals);
BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache,
last_insert_len, params, commands, num_literals);
BROTLI_FREE(m, nodes);
}
void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params,
HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals) {
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
@@ -767,8 +760,10 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches);
if (BROTLI_IS_OOM(m)) return;
num_found_matches = FindAllMatchesH10(hasher,
&params->dictionary, ringbuffer, ringbuffer_mask, pos, max_length,
max_distance, gap, params, &matches[cur_match_pos + shadow_matches]);
&params->dictionary,
ringbuffer, ringbuffer_mask, pos, max_length,
max_distance, gap, params,
&matches[cur_match_pos + shadow_matches]);
cur_match_end = cur_match_pos + num_found_matches;
for (j = cur_match_pos; j + 1 < cur_match_end; ++j) {
BROTLI_DCHECK(BackwardMatchLength(&matches[j]) <=
@@ -814,10 +809,10 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m,
*last_insert_len = orig_last_insert_len;
memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0]));
*num_commands += ZopfliIterate(num_bytes, position, ringbuffer,
ringbuffer_mask, params, max_backward_limit, gap, dist_cache,
&model, num_matches, matches, nodes);
BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit,
nodes, dist_cache, last_insert_len, params, commands, num_literals);
ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches,
nodes);
BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache,
last_insert_len, params, commands, num_literals);
}
CleanupZopfliCostModel(m, &model);
BROTLI_FREE(m, nodes);

91
C/brotli/br_decode.c
View File

@@ -6,10 +6,6 @@
#include "decode.h"
#if defined(__ARM_NEON__)
#include <arm_neon.h>
#endif
#include <stdlib.h> /* free, malloc */
#include <string.h> /* memcpy, memset */
@@ -24,6 +20,10 @@
#include "./dec/prefix.h"
#include "./dec/state.h"
#if defined(BROTLI_TARGET_NEON)
#include <arm_neon.h>
#endif
#if defined(__cplusplus) || defined(c_plusplus)
extern "C" {
#endif
@@ -167,7 +167,7 @@ static BrotliDecoderErrorCode DecodeWindowBits(BrotliDecoderState* s,
}
static BROTLI_INLINE void memmove16(uint8_t* dst, uint8_t* src) {
#if defined(__ARM_NEON__)
#if defined(BROTLI_TARGET_NEON)
vst1q_u8(dst, vld1q_u8(src));
#else
uint32_t buffer[4];
@@ -347,15 +347,17 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE DecodeMetaBlockLength(
static BROTLI_INLINE uint32_t DecodeSymbol(uint32_t bits,
const HuffmanCode* table,
BrotliBitReader* br) {
table += bits & HUFFMAN_TABLE_MASK;
if (table->bits > HUFFMAN_TABLE_BITS) {
uint32_t nbits = table->bits - HUFFMAN_TABLE_BITS;
BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table);
BROTLI_HC_ADJUST_TABLE_INDEX(table, bits & HUFFMAN_TABLE_MASK);
if (BROTLI_HC_FAST_LOAD_BITS(table) > HUFFMAN_TABLE_BITS) {
uint32_t nbits = BROTLI_HC_FAST_LOAD_BITS(table) - HUFFMAN_TABLE_BITS;
BrotliDropBits(br, HUFFMAN_TABLE_BITS);
table += table->value;
table += (bits >> HUFFMAN_TABLE_BITS) & BitMask(nbits);
BROTLI_HC_ADJUST_TABLE_INDEX(table,
BROTLI_HC_FAST_LOAD_VALUE(table) +
((bits >> HUFFMAN_TABLE_BITS) & BitMask(nbits)));
}
BrotliDropBits(br, table->bits);
return table->value;
BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(table));
return BROTLI_HC_FAST_LOAD_VALUE(table);
}
/* Reads and decodes the next Huffman code from bit-stream.
@@ -371,19 +373,20 @@ static BROTLI_NOINLINE BROTLI_BOOL SafeDecodeSymbol(
const HuffmanCode* table, BrotliBitReader* br, uint32_t* result) {
uint32_t val;
uint32_t available_bits = BrotliGetAvailableBits(br);
BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table);
if (available_bits == 0) {
if (table->bits == 0) {
*result = table->value;
if (BROTLI_HC_FAST_LOAD_BITS(table) == 0) {
*result = BROTLI_HC_FAST_LOAD_VALUE(table);
return BROTLI_TRUE;
}
return BROTLI_FALSE; /* No valid bits at all. */
}
val = (uint32_t)BrotliGetBitsUnmasked(br);
table += val & HUFFMAN_TABLE_MASK;
if (table->bits <= HUFFMAN_TABLE_BITS) {
if (table->bits <= available_bits) {
BrotliDropBits(br, table->bits);
*result = table->value;
BROTLI_HC_ADJUST_TABLE_INDEX(table, val & HUFFMAN_TABLE_MASK);
if (BROTLI_HC_FAST_LOAD_BITS(table) <= HUFFMAN_TABLE_BITS) {
if (BROTLI_HC_FAST_LOAD_BITS(table) <= available_bits) {
BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(table));
*result = BROTLI_HC_FAST_LOAD_VALUE(table);
return BROTLI_TRUE;
} else {
return BROTLI_FALSE; /* Not enough bits for the first level. */
@@ -394,15 +397,15 @@ static BROTLI_NOINLINE BROTLI_BOOL SafeDecodeSymbol(
}
/* Speculatively drop HUFFMAN_TABLE_BITS. */
val = (val & BitMask(table->bits)) >> HUFFMAN_TABLE_BITS;
val = (val & BitMask(BROTLI_HC_FAST_LOAD_BITS(table))) >> HUFFMAN_TABLE_BITS;
available_bits -= HUFFMAN_TABLE_BITS;
table += table->value + val;
if (available_bits < table->bits) {
BROTLI_HC_ADJUST_TABLE_INDEX(table, BROTLI_HC_FAST_LOAD_VALUE(table) + val);
if (available_bits < BROTLI_HC_FAST_LOAD_BITS(table)) {
return BROTLI_FALSE; /* Not enough bits for the second level. */
}
BrotliDropBits(br, HUFFMAN_TABLE_BITS + table->bits);
*result = table->value;
BrotliDropBits(br, HUFFMAN_TABLE_BITS + BROTLI_HC_FAST_LOAD_BITS(table));
*result = BROTLI_HC_FAST_LOAD_VALUE(table);
return BROTLI_TRUE;
}
@@ -425,9 +428,10 @@ static BROTLI_INLINE void PreloadSymbol(int safe,
if (safe) {
return;
}
table += BrotliGetBits(br, HUFFMAN_TABLE_BITS);
*bits = table->bits;
*value = table->value;
BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table);
BROTLI_HC_ADJUST_TABLE_INDEX(table, BrotliGetBits(br, HUFFMAN_TABLE_BITS));
*bits = BROTLI_HC_FAST_LOAD_BITS(table);
*value = BROTLI_HC_FAST_LOAD_VALUE(table);
}
/* Decodes the next Huffman code using data prepared by PreloadSymbol.
@@ -441,10 +445,11 @@ static BROTLI_INLINE uint32_t ReadPreloadedSymbol(const HuffmanCode* table,
uint32_t val = BrotliGet16BitsUnmasked(br);
const HuffmanCode* ext = table + (val & HUFFMAN_TABLE_MASK) + *value;
uint32_t mask = BitMask((*bits - HUFFMAN_TABLE_BITS));
BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(ext);
BrotliDropBits(br, HUFFMAN_TABLE_BITS);
ext += (val >> HUFFMAN_TABLE_BITS) & mask;
BrotliDropBits(br, ext->bits);
result = ext->value;
BROTLI_HC_ADJUST_TABLE_INDEX(ext, (val >> HUFFMAN_TABLE_BITS) & mask);
BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(ext));
result = BROTLI_HC_FAST_LOAD_VALUE(ext);
} else {
BrotliDropBits(br, *bits);
}
@@ -597,6 +602,7 @@ static BrotliDecoderErrorCode ReadSymbolCodeLengths(
while (symbol < alphabet_size && space > 0) {
const HuffmanCode* p = s->table;
uint32_t code_len;
BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(p);
if (!BrotliCheckInputAmount(br, BROTLI_SHORT_FILL_BIT_WINDOW_READ)) {
s->symbol = symbol;
s->repeat = repeat;
@@ -606,10 +612,10 @@ static BrotliDecoderErrorCode ReadSymbolCodeLengths(
return BROTLI_DECODER_NEEDS_MORE_INPUT;
}
BrotliFillBitWindow16(br);
p += BrotliGetBitsUnmasked(br) &
BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH);
BrotliDropBits(br, p->bits); /* Use 1..5 bits. */
code_len = p->value; /* code_len == 0..17 */
BROTLI_HC_ADJUST_TABLE_INDEX(p, BrotliGetBitsUnmasked(br) &
BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH));
BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(p)); /* Use 1..5 bits. */
code_len = BROTLI_HC_FAST_LOAD_VALUE(p); /* code_len == 0..17 */
if (code_len < BROTLI_REPEAT_PREVIOUS_CODE_LENGTH) {
ProcessSingleCodeLength(code_len, &symbol, &repeat, &space,
&prev_code_len, symbol_lists, code_length_histo, next_symbol);
@@ -637,31 +643,34 @@ static BrotliDecoderErrorCode SafeReadSymbolCodeLengths(
uint32_t code_len;
uint32_t available_bits;
uint32_t bits = 0;
BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(p);
if (get_byte && !BrotliPullByte(br)) return BROTLI_DECODER_NEEDS_MORE_INPUT;
get_byte = BROTLI_FALSE;
available_bits = BrotliGetAvailableBits(br);
if (available_bits != 0) {
bits = (uint32_t)BrotliGetBitsUnmasked(br);
}
p += bits & BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH);
if (p->bits > available_bits) {
BROTLI_HC_ADJUST_TABLE_INDEX(p,
bits & BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH));
if (BROTLI_HC_FAST_LOAD_BITS(p) > available_bits) {
get_byte = BROTLI_TRUE;
continue;
}
code_len = p->value; /* code_len == 0..17 */
code_len = BROTLI_HC_FAST_LOAD_VALUE(p); /* code_len == 0..17 */
if (code_len < BROTLI_REPEAT_PREVIOUS_CODE_LENGTH) {
BrotliDropBits(br, p->bits);
BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(p));
ProcessSingleCodeLength(code_len, &s->symbol, &s->repeat, &s->space,
&s->prev_code_len, s->symbol_lists, s->code_length_histo,
s->next_symbol);
} else { /* code_len == 16..17, extra_bits == 2..3 */
uint32_t extra_bits = code_len - 14U;
uint32_t repeat_delta = (bits >> p->bits) & BitMask(extra_bits);
if (available_bits < p->bits + extra_bits) {
uint32_t repeat_delta = (bits >> BROTLI_HC_FAST_LOAD_BITS(p)) &
BitMask(extra_bits);
if (available_bits < BROTLI_HC_FAST_LOAD_BITS(p) + extra_bits) {
get_byte = BROTLI_TRUE;
continue;
}
BrotliDropBits(br, p->bits + extra_bits);
BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(p) + extra_bits);
ProcessRepeatedCodeLength(code_len, repeat_delta, alphabet_size,
&s->symbol, &s->repeat, &s->space, &s->prev_code_len,
&s->repeat_code_len, s->symbol_lists, s->code_length_histo,

34
C/brotli/br_encode.c
View File

@@ -496,6 +496,8 @@ static void DecideOverLiteralContextModeling(const uint8_t* input,
static BROTLI_BOOL ShouldCompress(
const uint8_t* data, const size_t mask, const uint64_t last_flush_pos,
const size_t bytes, const size_t num_literals, const size_t num_commands) {
/* TODO: find more precise minimal block overhead. */
if (bytes <= 2) return BROTLI_FALSE;
if (num_commands < (bytes >> 8) + 2) {
if (num_literals > 0.99 * (double)bytes) {
uint32_t literal_histo[256] = { 0 };
@@ -674,12 +676,14 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) {
if (BROTLI_IS_OOM(&s->memory_manager_)) return BROTLI_FALSE;
if (s->is_initialized_) return BROTLI_TRUE;
s->last_bytes_bits_ = 0;
s->last_bytes_ = 0;
s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX;
SanitizeParams(&s->params);
s->params.lgblock = ComputeLgBlock(&s->params);
ChooseDistanceParams(&s->params);
s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX;
RingBufferSetup(&s->params, &s->ringbuffer_);
/* Initialize last byte with stream header. */
@@ -1029,23 +1033,20 @@ static BROTLI_BOOL EncodeData(
if (s->params.quality == ZOPFLIFICATION_QUALITY) {
BROTLI_DCHECK(s->params.hasher.type == 10);
BrotliCreateZopfliBackwardReferences(m,
bytes, wrapped_last_processed_pos,
BrotliCreateZopfliBackwardReferences(m, bytes, wrapped_last_processed_pos,
data, mask, &s->params, s->hasher_, s->dist_cache_,
&s->last_insert_len_, &s->commands_[s->num_commands_],
&s->num_commands_, &s->num_literals_);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
} else if (s->params.quality == HQ_ZOPFLIFICATION_QUALITY) {
BROTLI_DCHECK(s->params.hasher.type == 10);
BrotliCreateHqZopfliBackwardReferences(m,
bytes, wrapped_last_processed_pos,
BrotliCreateHqZopfliBackwardReferences(m, bytes, wrapped_last_processed_pos,
data, mask, &s->params, s->hasher_, s->dist_cache_,
&s->last_insert_len_, &s->commands_[s->num_commands_],
&s->num_commands_, &s->num_literals_);
if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
} else {
BrotliCreateBackwardReferences(
bytes, wrapped_last_processed_pos,
BrotliCreateBackwardReferences(bytes, wrapped_last_processed_pos,
data, mask, &s->params, s->hasher_, s->dist_cache_,
&s->last_insert_len_, &s->commands_[s->num_commands_],
&s->num_commands_, &s->num_literals_);
@@ -1166,7 +1167,6 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
MemoryManager* m = &memory_manager;
const size_t mask = BROTLI_SIZE_MAX >> 1;
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(lgwin);
int dist_cache[4] = { 4, 11, 15, 16 };
int saved_dist_cache[4] = { 4, 11, 15, 16 };
BROTLI_BOOL ok = BROTLI_TRUE;
@@ -1176,8 +1176,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
uint8_t last_bytes_bits;
HasherHandle hasher = NULL;
const size_t hasher_eff_size =
BROTLI_MIN(size_t, input_size, max_backward_limit + BROTLI_WINDOW_GAP);
const size_t hasher_eff_size = BROTLI_MIN(size_t,
input_size, BROTLI_MAX_BACKWARD_LIMIT(lgwin) + BROTLI_WINDOW_GAP);
BrotliEncoderParams params;
@@ -1238,9 +1238,9 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
BrotliInitZopfliNodes(nodes, block_size + 1);
StitchToPreviousBlockH10(hasher, block_size, block_start,
input_buffer, mask);
path_size = BrotliZopfliComputeShortestPath(m,
block_size, block_start, input_buffer, mask, &params,
max_backward_limit, dist_cache, hasher, nodes);
path_size = BrotliZopfliComputeShortestPath(m, block_size, block_start,
input_buffer, mask, &params, dist_cache, hasher,
nodes);
if (BROTLI_IS_OOM(m)) goto oom;
/* We allocate a command buffer in the first iteration of this loop that
will be likely big enough for the whole metablock, so that for most
@@ -1262,10 +1262,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10(
}
commands = new_commands;
}
BrotliZopfliCreateCommands(block_size, block_start, max_backward_limit,
&nodes[0], dist_cache, &last_insert_len,
&params, &commands[num_commands],
&num_literals);
BrotliZopfliCreateCommands(block_size, block_start, &nodes[0], dist_cache,
&last_insert_len, &params, &commands[num_commands], &num_literals);
num_commands += path_size;
block_start += block_size;
metablock_size += block_size;

77
C/brotli/br_huffman.c
View File

@@ -142,8 +142,7 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
/* Special case: all symbols but one have 0 code length. */
if (offset[0] == 0) {
code.bits = 0;
code.value = (uint16_t)sorted[0];
code = ConstructHuffmanCode(0, (uint16_t)sorted[0]);
for (key = 0; key < (brotli_reg_t)table_size; ++key) {
table[key] = code;
}
@@ -157,9 +156,8 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table,
bits = 1;
step = 2;
do {
code.bits = (uint8_t)bits;
for (bits_count = count[bits]; bits_count != 0; --bits_count) {
code.value = (uint16_t)sorted[symbol++];
code = ConstructHuffmanCode((uint8_t)bits, (uint16_t)sorted[symbol++]);
ReplicateValue(&table[BrotliReverseBits(key)], step, table_size, code);
key += key_step;
}
@@ -211,11 +209,10 @@ uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
bits = 1;
step = 2;
do {
code.bits = (uint8_t)bits;
symbol = bits - (BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1);
for (bits_count = count[bits]; bits_count != 0; --bits_count) {
symbol = symbol_lists[symbol];
code.value = (uint16_t)symbol;
code = ConstructHuffmanCode((uint8_t)bits, (uint16_t)symbol);
ReplicateValue(&table[BrotliReverseBits(key)], step, table_size, code);
key += key_step;
}
@@ -244,14 +241,13 @@ uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table,
total_size += table_size;
sub_key = BrotliReverseBits(key);
key += key_step;
root_table[sub_key].bits = (uint8_t)(table_bits + root_bits);
root_table[sub_key].value =
(uint16_t)(((size_t)(table - root_table)) - sub_key);
root_table[sub_key] = ConstructHuffmanCode(
(uint8_t)(table_bits + root_bits),
(uint16_t)(((size_t)(table - root_table)) - sub_key));
sub_key = 0;
}
code.bits = (uint8_t)(len - root_bits);
symbol = symbol_lists[symbol];
code.value = (uint16_t)symbol;
code = ConstructHuffmanCode((uint8_t)(len - root_bits), (uint16_t)symbol);
ReplicateValue(
&table[BrotliReverseBits(sub_key)], step, table_size, code);
sub_key += sub_key_step;
@@ -270,35 +266,28 @@ uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
const uint32_t goal_size = 1U << root_bits;
switch (num_symbols) {
case 0:
table[0].bits = 0;
table[0].value = val[0];
table[0] = ConstructHuffmanCode(0, val[0]);
break;
case 1:
table[0].bits = 1;
table[1].bits = 1;
if (val[1] > val[0]) {
table[0].value = val[0];
table[1].value = val[1];
table[0] = ConstructHuffmanCode(1, val[0]);
table[1] = ConstructHuffmanCode(1, val[1]);
} else {
table[0].value = val[1];
table[1].value = val[0];
table[0] = ConstructHuffmanCode(1, val[1]);
table[1] = ConstructHuffmanCode(1, val[0]);
}
table_size = 2;
break;
case 2:
table[0].bits = 1;
table[0].value = val[0];
table[2].bits = 1;
table[2].value = val[0];
table[0] = ConstructHuffmanCode(1, val[0]);
table[2] = ConstructHuffmanCode(1, val[0]);
if (val[2] > val[1]) {
table[1].value = val[1];
table[3].value = val[2];
table[1] = ConstructHuffmanCode(2, val[1]);
table[3] = ConstructHuffmanCode(2, val[2]);
} else {
table[1].value = val[2];
table[3].value = val[1];
table[1] = ConstructHuffmanCode(2, val[2]);
table[3] = ConstructHuffmanCode(2, val[1]);
}
table[1].bits = 2;
table[3].bits = 2;
table_size = 4;
break;
case 3: {
@@ -312,33 +301,27 @@ uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table,
}
}
}
for (i = 0; i < 4; ++i) {
table[i].bits = 2;
}
table[0].value = val[0];
table[2].value = val[1];
table[1].value = val[2];
table[3].value = val[3];
table[0] = ConstructHuffmanCode(2, val[0]);
table[2] = ConstructHuffmanCode(2, val[1]);
table[1] = ConstructHuffmanCode(2, val[2]);
table[3] = ConstructHuffmanCode(2, val[3]);
table_size = 4;
break;
}
case 4: {
int i;
if (val[3] < val[2]) {
uint16_t t = val[3];
val[3] = val[2];
val[2] = t;
}
for (i = 0; i < 7; ++i) {
table[i].value = val[0];
table[i].bits = (uint8_t)(1 + (i & 1));
}
table[1].value = val[1];
table[3].value = val[2];
table[5].value = val[1];
table[7].value = val[3];
table[3].bits = 3;
table[7].bits = 3;
table[0] = ConstructHuffmanCode(1, val[0]);
table[1] = ConstructHuffmanCode(2, val[1]);
table[2] = ConstructHuffmanCode(1, val[0]);
table[3] = ConstructHuffmanCode(3, val[2]);
table[4] = ConstructHuffmanCode(1, val[0]);
table[5] = ConstructHuffmanCode(2, val[1]);
table[6] = ConstructHuffmanCode(1, val[0]);
table[7] = ConstructHuffmanCode(3, val[3]);
table_size = 8;
break;
}

8
C/brotli/br_transform.c
View File

@@ -191,11 +191,11 @@ static int ToUpperCase(uint8_t* p) {
}
int BrotliTransformDictionaryWord(uint8_t* dst, const uint8_t* word, int len,
const BrotliTransforms* transforms, int transfom_idx) {
const BrotliTransforms* transforms, int transform_idx) {
int idx = 0;
const uint8_t* prefix = BROTLI_TRANSFORM_PREFIX(transforms, transfom_idx);
uint8_t type = BROTLI_TRANSFORM_TYPE(transforms, transfom_idx);
const uint8_t* suffix = BROTLI_TRANSFORM_SUFFIX(transforms, transfom_idx);
const uint8_t* prefix = BROTLI_TRANSFORM_PREFIX(transforms, transform_idx);
uint8_t type = BROTLI_TRANSFORM_TYPE(transforms, transform_idx);
const uint8_t* suffix = BROTLI_TRANSFORM_SUFFIX(transforms, transform_idx);
{
int prefix_len = *prefix++;
while (prefix_len--) { dst[idx++] = *prefix++; }

14
C/brotli/common/platform.h
View File

@@ -71,7 +71,7 @@ OR:
*/
#if BROTLI_GNUC_HAS_BUILTIN(__builtin_expect, 3, 0, 0) || \
BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \
BROTLI_SUNPRO_VERSION_CHECK(5, 12, 0) || \
BROTLI_SUNPRO_VERSION_CHECK(5, 15, 0) || \
BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \
BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \
BROTLI_TI_VERSION_CHECK(7, 3, 0) || \
@@ -180,6 +180,12 @@ OR:
#define BROTLI_UNUSED_FUNCTION static BROTLI_INLINE
#endif
#if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)
#define BROTLI_ALIGNED(N) __attribute__((aligned(N)))
#else
#define BROTLI_ALIGNED(N)
#endif
#if (defined(__ARM_ARCH) && (__ARM_ARCH == 7)) || \
(defined(M_ARM) && (M_ARM == 7))
#define BROTLI_TARGET_ARMV7
@@ -197,6 +203,10 @@ OR:
#endif /* ARMv8 */
#if defined(__ARM_NEON__) || defined(__ARM_NEON)
#define BROTLI_TARGET_NEON
#endif
#if defined(__i386) || defined(_M_IX86)
#define BROTLI_TARGET_X86
#endif
@@ -343,7 +353,7 @@ static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) {
/* If __attribute__(aligned) is available, use that. Otherwise, memcpy. */
#if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)
typedef __attribute__((aligned(1))) uint64_t brotli_unaligned_uint64_t;
typedef BROTLI_ALIGNED(1) uint64_t brotli_unaligned_uint64_t;
static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) {
return (uint64_t) ((brotli_unaligned_uint64_t*) p)[0];

4
C/brotli/common/version.h
View File

@@ -14,13 +14,13 @@
BrotliEncoderVersion methods. */
/* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */
#define BROTLI_VERSION 0x1000006
#define BROTLI_VERSION 0x1000007
/* This macro is used by build system to produce Libtool-friendly soname. See
https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html
*/
/* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */
#define BROTLI_ABI_VERSION 0x1006000
#define BROTLI_ABI_VERSION 0x1007000
#endif /* BROTLI_COMMON_VERSION_H_ */

55
C/brotli/dec/huffman.h
View File

@@ -33,11 +33,66 @@ static const uint16_t kMaxHuffmanTableSize[] = {
#define BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH 5
#if ((defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_32)) && \
BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0))
#define BROTLI_HUFFMAN_CODE_FAST_LOAD
#endif
#if !defined(BROTLI_HUFFMAN_CODE_FAST_LOAD)
/* Do not create this struct directly - use the ConstructHuffmanCode
* constructor below! */
typedef struct {
uint8_t bits; /* number of bits used for this symbol */
uint16_t value; /* symbol value or table offset */
} HuffmanCode;
static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits,
const uint16_t value) {
HuffmanCode h;
h.bits = bits;
h.value = value;
return h;
}
/* Please use the following macros to optimize HuffmanCode accesses in hot
* paths.
*
* For example, assuming |table| contains a HuffmanCode pointer:
*
* BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table);
* BROTLI_HC_ADJUST_TABLE_INDEX(table, index_into_table);
* *bits = BROTLI_HC_GET_BITS(table);
* *value = BROTLI_HC_GET_VALUE(table);
* BROTLI_HC_ADJUST_TABLE_INDEX(table, offset);
* *bits2 = BROTLI_HC_GET_BITS(table);
* *value2 = BROTLI_HC_GET_VALUE(table);
*
*/
#define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H)
#define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V)
/* These must be given a HuffmanCode pointer! */
#define BROTLI_HC_FAST_LOAD_BITS(H) (H->bits)
#define BROTLI_HC_FAST_LOAD_VALUE(H) (H->value)
#else /* BROTLI_HUFFMAN_CODE_FAST_LOAD */
typedef BROTLI_ALIGNED(4) uint32_t HuffmanCode;
static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits,
const uint16_t value) {
return ((value & 0xFFFF) << 16) | (bits & 0xFF);
}
#define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) uint32_t __fastload_##H = (*H)
#define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V); __fastload_##H = (*H)
/* These must be given a HuffmanCode pointer! */
#define BROTLI_HC_FAST_LOAD_BITS(H) ((__fastload_##H) & 0xFF)
#define BROTLI_HC_FAST_LOAD_VALUE(H) ((__fastload_##H) >> 16)
#endif /* BROTLI_HUFFMAN_CODE_FAST_LOAD */
/* Builds Huffman lookup table assuming code lengths are in symbol order. */
BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table,
const uint8_t* const code_lengths, uint16_t* count);

13
C/brotli/enc/backward_references_hq.h
View File

@@ -74,15 +74,14 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length);
(1) nodes[i].copy_length() >= 2
(2) nodes[i].command_length() <= i and
(3) nodes[i - nodes[i].command_length()].cost < kInfinity */
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(MemoryManager* m,
size_t num_bytes, size_t position, const uint8_t* ringbuffer,
size_t ringbuffer_mask, const BrotliEncoderParams* params,
const size_t max_backward_limit, const int* dist_cache, HasherHandle hasher,
ZopfliNode* nodes);
BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(
MemoryManager* m, size_t num_bytes,
size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params,
const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes);
BROTLI_INTERNAL void BrotliZopfliCreateCommands(
const size_t num_bytes, const size_t block_start,
const size_t max_backward_limit, const ZopfliNode* nodes,
const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes,
int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params,
Command* commands, size_t* num_literals);

18
C/brotli/enc/backward_references_inc.h
View File

@@ -10,9 +10,9 @@
static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
size_t num_bytes, size_t position,
const uint8_t* ringbuffer, size_t ringbuffer_mask,
const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache,
size_t* last_insert_len, Command* commands, size_t* num_commands,
size_t* num_literals) {
const BrotliEncoderParams* params,
HasherHandle hasher, int* dist_cache, size_t* last_insert_len,
Command* commands, size_t* num_commands, size_t* num_literals) {
/* Set maximum distance, see section 9.1. of the spec. */
const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin);
@@ -42,9 +42,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
sr.distance = 0;
sr.score = kMinScore;
FN(FindLongestMatch)(hasher, &params->dictionary,
ringbuffer, ringbuffer_mask, dist_cache, position,
max_length, max_distance, gap,
params->dist.max_distance, &sr);
ringbuffer, ringbuffer_mask, dist_cache, position, max_length,
max_distance, gap, params->dist.max_distance, &sr);
if (sr.score > kMinScore) {
/* Found a match. Let's look for something even better ahead. */
int delayed_backward_references_in_row = 0;
@@ -58,7 +57,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
sr2.distance = 0;
sr2.score = kMinScore;
max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit);
FN(FindLongestMatch)(hasher, &params->dictionary,
FN(FindLongestMatch)(hasher,
&params->dictionary,
ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length,
max_distance, gap, params->dist.max_distance, &sr2);
if (sr2.score >= sr.score + cost_diff_lazy) {
@@ -80,8 +80,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
{
/* The first 16 codes are special short-codes,
and the minimum offset is 1. */
size_t distance_code =
ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache);
size_t distance_code = ComputeDistanceCode(
sr.distance, max_distance + gap, dist_cache);
if ((sr.distance <= (max_distance + gap)) && distance_code > 0) {
dist_cache[3] = dist_cache[2];
dist_cache[2] = dist_cache[1];

9
C/brotli/enc/hash.h
View File

@@ -149,9 +149,9 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance(
}
static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem(
const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data,
size_t max_length, size_t max_backward, size_t max_distance,
HasherSearchResult* out) {
const BrotliEncoderDictionary* dictionary, size_t item,
const uint8_t* data, size_t max_length, size_t max_backward,
size_t max_distance, HasherSearchResult* out) {
size_t len;
size_t word_idx;
size_t offset;
@@ -208,7 +208,8 @@ static BROTLI_INLINE void SearchInStaticDictionary(
self->dict_num_lookups++;
if (item != 0) {
BROTLI_BOOL item_matches = TestStaticDictionaryItem(
dictionary, item, data, max_length, max_backward, max_distance, out);
dictionary, item, data,
max_length, max_backward, max_distance, out);
if (item_matches) {
self->dict_num_matches++;
}

11
C/brotli/enc/hash_composite_inc.h
View File

@@ -121,13 +121,16 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
const size_t max_length, const size_t max_backward,
const size_t gap, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
HashComposite* self = FN(Self)(handle);
FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask,
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
distance_cache, cur_ix, max_length, max_backward, gap,
max_distance, out);
FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask,
distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out);
distance_cache, cur_ix, max_length, max_backward, gap,
max_distance, out);
}
#undef HashComposite

5
C/brotli/enc/hash_longest_match64_inc.h
View File

@@ -161,8 +161,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
const size_t max_length, const size_t max_backward,
const size_t gap, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);

5
C/brotli/enc/hash_longest_match_inc.h
View File

@@ -154,8 +154,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
const size_t max_length, const size_t max_backward,
const size_t gap, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
HasherCommon* common = GetHasherCommon(handle);
HashLongestMatch* self = FN(Self)(handle);
uint16_t* num = FN(Num)(self);

5
C/brotli/enc/hash_rolling_inc.h
View File

@@ -155,8 +155,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary,
const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask,
const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) {
const size_t max_length, const size_t max_backward,
const size_t gap, const size_t max_distance,
HasherSearchResult* BROTLI_RESTRICT out) {
HashRolling* self = FN(Self)(handle);
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
size_t pos = self->next_ix;

5
C/brotli/enc/hash_to_binary_tree_inc.h
View File

@@ -202,8 +202,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)(
static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle,
const BrotliEncoderDictionary* dictionary, const uint8_t* data,
const size_t ring_buffer_mask, const size_t cur_ix,
const size_t max_length, const size_t max_backward, const size_t gap,
const BrotliEncoderParams* params, BackwardMatch* matches) {
const size_t max_length, const size_t max_backward,
const size_t gap, const BrotliEncoderParams* params,
BackwardMatch* matches) {
BackwardMatch* const orig_matches = matches;
const size_t cur_ix_masked = cur_ix & ring_buffer_mask;
size_t best_len = 1;

4
C/hashes/README.md Normal file
View File

@@ -0,0 +1,4 @@
Taken from lib/crypto/* of samba-4.5.8.tar.gz.
/TR 2018-11-15

108
C/zstd/README.md
View File

@@ -7,13 +7,32 @@ in order to make it easier to select or exclude features.
#### Building
`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions),
including commands variables, staged install, directory variables and standard targets.
- `make` : generates both static and dynamic libraries
- `make install` : install libraries in default system directories
- `make install` : install libraries and headers in target system directories
`libzstd` default scope includes compression, decompression, dictionary building,
and decoding support for legacy formats >= v0.5.0.
`libzstd` default scope is pretty large, including compression, decompression, dictionary builder,
and support for decoding legacy formats >= v0.5.0.
The scope can be reduced on demand (see paragraph _modular build_).
#### Multithreading support
Multithreading is disabled by default when building with `make`.
Enabling multithreading requires 2 conditions :
- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`)
- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
Both conditions are automatically applied when invoking `make lib-mt` target.
When linking a POSIX program with a multithreaded version of `libzstd`,
note that it's necessary to request the `-pthread` flag during link stage.
Multithreading capabilities are exposed
via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592).
This API is still labelled experimental,
but is expected to become "stable" in the near future.
#### API
@@ -26,63 +45,70 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
Optional advanced features are exposed via :
- `lib/common/zstd_errors.h` : translates `size_t` function results
into an `ZSTD_ErrorCode`, for accurate error handling.
into a `ZSTD_ErrorCode`, for accurate error handling.
- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
it unlocks access to advanced experimental API,
exposed in second part of `zstd.h`.
These APIs are not "stable", their definition may change in the future.
As a consequence, it shall ___never be used with dynamic library___ !
it unlocks access to the experimental API,
exposed in the second part of `zstd.h`.
All definitions in the experimental APIs are unstable,
they may still change in the future, or even be removed.
As a consequence, experimental definitions shall ___never be used with dynamic library___ !
Only static linking is allowed.
#### Modular build
It's possible to compile only a limited set of features.
It's possible to compile only a limited set of features within `libzstd`.
The file structure is designed to make this selection manually achievable for any build system :
- Directory `lib/common` is always required, for all variants.
- Compression source code lies in `lib/compress`
- Decompression source code lies in `lib/decompress`
- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
The API is exposed in `lib/dictBuilder/zdict.h`.
This module depends on both `lib/common` and `lib/compress` .
- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`.
- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`.
This module depends on `lib/common` and `lib/decompress`.
To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation.
Specifying a number limits versions supported to that version onward.
For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0".
`ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on.
Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`.
It can be changed at build by any other value.
Note that any number >= 8 translates into "do __not__ support legacy formats",
since all versions of `zstd` >= v0.8 are compatible with v1+ specification.
`ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats".
Once enabled, this capability is transparently triggered within decompression functions.
It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`.
Each version also provides an additional dedicated set of advanced API.
Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats".
By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`.
Decoding supported legacy format is a transparent capability triggered within decompression functions.
It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`.
Each version does also provide its own set of advanced API.
For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
Note : `lib/legacy` only supports _decoding_ legacy formats.
- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will
also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
dictBuilder).
- While invoking `make libzstd`, it's possible to define build macros
`ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
This will also disable compilation of all dependencies
(eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
#### Multithreading support
- There are some additional build macros that can be used to minify the decoder.
Multithreading is disabled by default when building with `make`.
Enabling multithreading requires 2 conditions :
- set macro `ZSTD_MULTITHREAD`
- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`)
Zstandard often has more than one implementation of a piece of functionality,
where each implementation optimizes for different scenarios. For example, the
Huffman decoder has complementary implementations that decode the stream one
symbol at a time or two symbols at a time. Zstd normally includes both (and
dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1`
or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
only one or the other of two decompression implementations. The smallest
binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
`ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`.
Both conditions are automatically triggered by invoking `make lib-mt` target.
Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
it's necessary to trigger `-pthread` flag during link stage.
Multithreading capabilities are exposed
via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919).
This API is still considered experimental,
but is expected to become "stable" at some point in the future.
For squeezing the last ounce of size out, you can also define
`ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
which removes the error messages that are otherwise returned by
`ZSTD_getErrorName`.
#### Windows : using MinGW+MSYS to create DLL
@@ -113,8 +139,8 @@ Consider migrating code towards supported streaming API exposed in `zstd.h`.
The other files are not source code. There are :
- `LICENSE` : contains the BSD license text
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
- `BUCK` : support for `buck` build system (https://buckbuild.com/)
- `libzstd.pc.in` : for `pkg-config` (used in `make install`)
- `Makefile` : `make` script to build and install zstd library (static and dynamic)
- `README.md` : this file
- `dll/` : resources directory for Windows compilation
- `libzstd.pc.in` : script for `pkg-config` (used in `make install`)

4
C/zstd/bitstream.h
View File

@@ -389,7 +389,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits)
* Read (consume) next n bits from local register and update.
* Pay attention to not read more than nbBits contained into local register.
* @return : extracted value. */
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits)
{
size_t const value = BIT_lookBits(bitD, nbBits);
BIT_skipBits(bitD, nbBits);
@@ -398,7 +398,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits)
/*! BIT_readBitsFast() :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits)
{
size_t const value = BIT_lookBitsFast(bitD, nbBits);
assert(nbBits >= 1);

31
C/zstd/compiler.h
View File

@@ -15,6 +15,8 @@
* Compiler specifics
*********************************************************/
/* force inlining */
#if !defined(ZSTD_NO_INLINE)
#if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */
# define INLINE_KEYWORD inline
#else
@@ -29,6 +31,13 @@
# define FORCE_INLINE_ATTR
#endif
#else
#define INLINE_KEYWORD
#define FORCE_INLINE_ATTR
#endif
/**
* FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant
* parameters. They must be inlined for the compiler to elimininate the constant
@@ -89,23 +98,21 @@
#endif
/* prefetch
* can be disabled, by declaring NO_PREFETCH macro
* All prefetch invocations use a single default locality 2,
* generating instruction prefetcht1,
* which, according to Intel, means "load data into L2 cache".
* This is a good enough "middle ground" for the time being,
* though in theory, it would be better to specialize locality depending on data being prefetched.
* Tests could not determine any sensible difference based on locality value. */
* can be disabled, by declaring NO_PREFETCH build macro */
#if defined(NO_PREFETCH)
# define PREFETCH(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
#else
# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */
# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
# define PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1)
# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
# define PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */)
# else
# define PREFETCH(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */
# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */
# endif
#endif /* NO_PREFETCH */
@@ -116,7 +123,7 @@
size_t const _size = (size_t)(s); \
size_t _pos; \
for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \
PREFETCH(_ptr + _pos); \
PREFETCH_L2(_ptr + _pos); \
} \
}

2
C/zstd/cpu.h
View File

@@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) {
__asm__(
"pushl %%ebx\n\t"
"cpuid\n\t"
"movl %%ebx, %%eax\n\r"
"movl %%ebx, %%eax\n\t"
"popl %%ebx"
: "=a"(f7b), "=c"(f7c)
: "a"(7), "c"(0)

27
C/zstd/debug.h
View File

@@ -57,9 +57,9 @@ extern "C" {
#endif
/* static assert is triggered at compile time, leaving no runtime artefact,
* but can only work with compile-time constants.
* This variant can only be used inside a function. */
/* static assert is triggered at compile time, leaving no runtime artefact.
* static assert only works with compile-time constants.
* Also, this variant can only be used inside a function. */
#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1])
@@ -70,9 +70,19 @@ extern "C" {
# define DEBUGLEVEL 0
#endif
/* DEBUGFILE can be defined externally,
* typically through compiler command line.
* note : currently useless.
* Value must be stderr or stdout */
#ifndef DEBUGFILE
# define DEBUGFILE stderr
#endif
/* recommended values for DEBUGLEVEL :
* 0 : no debug, all run-time functions disabled
* 1 : no display, enables assert() only
* 0 : release mode, no debug, all run-time checks disabled
* 1 : enables assert() only, no display
* 2 : reserved, for currently active debug path
* 3 : events once per object lifetime (CCtx, CDict, etc.)
* 4 : events once per frame
@@ -81,7 +91,7 @@ extern "C" {
* 7+: events at every position (*very* verbose)
*
* It's generally inconvenient to output traces > 5.
* In which case, it's possible to selectively enable higher verbosity levels
* In which case, it's possible to selectively trigger high verbosity levels
* by modifying g_debug_level.
*/
@@ -95,10 +105,11 @@ extern "C" {
#if (DEBUGLEVEL>=2)
# include <stdio.h>
extern int g_debuglevel; /* here, this variable is only declared,
extern int g_debuglevel; /* the variable is only declared,
it actually lives in debug.c,
and is shared by the whole process.
It's typically used to enable very verbose levels
It's not thread-safe.
It's useful when enabling very verbose levels
on selective conditions (such as position in src) */
# define RAWLOG(l, ...) { \

6
C/zstd/error_private.c
View File

@@ -14,6 +14,10 @@
const char* ERR_getErrorString(ERR_enum code)
{
#ifdef ZSTD_STRIP_ERROR_STRINGS
(void)code;
return "Error strings stripped";
#else
static const char* const notErrorCode = "Unspecified error code";
switch( code )
{
@@ -39,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code)
case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples";
case PREFIX(dstSize_tooSmall): return "Destination buffer is too small";
case PREFIX(srcSize_wrong): return "Src size is incorrect";
case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer";
/* following error codes are not stable and may be removed or changed in a future version */
case PREFIX(frameIndex_tooLarge): return "Frame index is too large";
case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking";
case PREFIX(maxCode):
default: return notErrorCode;
}
#endif
}

4
C/zstd/fse.h
View File

@@ -512,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct)
const U32 tableLog = MEM_read16(ptr);
statePtr->value = (ptrdiff_t)1<<tableLog;
statePtr->stateTable = u16ptr+2;
statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1));
statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1);
statePtr->stateLog = tableLog;
}
@@ -531,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3
}
}
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol)
MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol)
{
FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol];
const U16* const stateTable = (const U16*)(statePtr->stateTable);

4
C/zstd/fse_compress.c
View File

@@ -658,7 +658,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 count[FSE_MAX_SYMBOL_VALUE+1];
unsigned count[FSE_MAX_SYMBOL_VALUE+1];
S16 norm[FSE_MAX_SYMBOL_VALUE+1];
FSE_CTable* CTable = (FSE_CTable*)workSpace;
size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue);
@@ -672,7 +672,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src
if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG;
/* Scan input and build symbol stats */
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) );
{ CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) );
if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */
if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */
if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */

30
C/zstd/hist.c
View File

@@ -73,6 +73,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
return largestCount;
}
typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e;
/* HIST_count_parallel_wksp() :
* store histogram into 4 intermediate tables, recombined at the end.
@@ -85,8 +86,8 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr,
static size_t HIST_count_parallel_wksp(
unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
unsigned checkMax,
unsigned* const workSpace)
HIST_checkInput_e check,
U32* const workSpace)
{
const BYTE* ip = (const BYTE*)source;
const BYTE* const iend = ip+sourceSize;
@@ -137,7 +138,7 @@ static size_t HIST_count_parallel_wksp(
/* finish last symbols */
while (ip<iend) Counting1[*ip++]++;
if (checkMax) { /* verify stats will fit into destination table */
if (check) { /* verify stats will fit into destination table */
U32 s; for (s=255; s>maxSymbolValue; s--) {
Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s];
if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall);
@@ -157,14 +158,18 @@ static size_t HIST_count_parallel_wksp(
/* HIST_countFast_wksp() :
* Same as HIST_countFast(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize,
unsigned* workSpace)
void* workSpace, size_t workSpaceSize)
{
if (sourceSize < 1500) /* heuristic threshold */
return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace);
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace);
}
/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */
@@ -172,24 +177,27 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize)
{
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters);
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters));
}
/* HIST_count_wksp() :
* Same as HIST_count(), but using an externally provided scratch buffer.
* `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* source, size_t sourceSize, unsigned* workSpace)
const void* source, size_t sourceSize,
void* workSpace, size_t workSpaceSize)
{
if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall);
if (*maxSymbolValuePtr < 255)
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace);
return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace);
*maxSymbolValuePtr = 255;
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace);
return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize);
}
size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize)
{
unsigned tmpCounters[HIST_WKSP_SIZE_U32];
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters);
return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters));
}

11
C/zstd/hist.h
View File

@@ -56,14 +56,16 @@ unsigned HIST_isError(size_t code); /**< tells if a return value is an error co
/* --- advanced histogram functions --- */
#define HIST_WKSP_SIZE_U32 1024
#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned))
/** HIST_count_wksp() :
* Same as HIST_count(), but using an externally provided scratch buffer.
* Benefit is this function will use very little stack space.
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize,
unsigned* workSpace);
void* workSpace, size_t workSpaceSize);
/** HIST_countFast() :
* same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr.
@@ -74,11 +76,12 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr,
/** HIST_countFast_wksp() :
* Same as HIST_countFast(), but using an externally provided scratch buffer.
* `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32
* `workSpace` is a writable buffer which must be 4-bytes aligned,
* `workSpaceSize` must be >= HIST_WKSP_SIZE
*/
size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr,
const void* src, size_t srcSize,
unsigned* workSpace);
void* workSpace, size_t workSpaceSize);
/*! HIST_count_simple() :
* Same as HIST_countFast(), this function is unsafe,

26
C/zstd/huf.h
View File

@@ -173,15 +173,19 @@ typedef U32 HUF_DTable;
* Advanced decompression functions
******************************************/
size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
#endif
size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */
size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */
size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
#endif
/* ****************************************
@@ -228,7 +232,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize,
#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1)
#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned))
size_t HUF_buildCTable_wksp (HUF_CElt* tree,
const U32* count, U32 maxSymbolValue, U32 maxNbBits,
const unsigned* count, U32 maxSymbolValue, U32 maxNbBits,
void* workSpace, size_t wkspSize);
/*! HUF_readStats() :
@@ -277,14 +281,22 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize);
#define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10)
#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32))
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize);
size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize);
#endif
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
/* ====================== */
@@ -306,24 +318,36 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2);
size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */
#endif
size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize);
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */
size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */
size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */
#endif
size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
#ifndef HUF_FORCE_DECOMPRESS_X1
size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable);
#endif
/* BMI2 variants.
* If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0.
*/
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);
#endif
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2);
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2);

52
C/zstd/huf_compress.c
View File

@@ -88,13 +88,13 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight
BYTE* op = ostart;
BYTE* const oend = ostart + dstSize;
U32 maxSymbolValue = HUF_TABLELOG_MAX;
unsigned maxSymbolValue = HUF_TABLELOG_MAX;
U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER;
FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)];
BYTE scratchBuffer[1<<MAX_FSE_TABLELOG_FOR_HUFF_HEADER];
U32 count[HUF_TABLELOG_MAX+1];
unsigned count[HUF_TABLELOG_MAX+1];
S16 norm[HUF_TABLELOG_MAX+1];
/* init conditions */
@@ -134,7 +134,7 @@ struct HUF_CElt_s {
`CTable` : Huffman tree to save, using huf representation.
@return : size of saved CTable */
size_t HUF_writeCTable (void* dst, size_t maxDstSize,
const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog)
const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog)
{
BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */
BYTE huffWeight[HUF_SYMBOLVALUE_MAX];
@@ -169,7 +169,7 @@ size_t HUF_writeCTable (void* dst, size_t maxDstSize,
}
size_t HUF_readCTable (HUF_CElt* CTable, U32* maxSymbolValuePtr, const void* src, size_t srcSize)
size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize)
{
BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; /* init not required, even though some static analyzer may complain */
U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */
@@ -315,7 +315,7 @@ typedef struct {
U32 current;
} rankPos;
static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
static void HUF_sort(nodeElt* huffNode, const unsigned* count, U32 maxSymbolValue)
{
rankPos rank[32];
U32 n;
@@ -347,7 +347,7 @@ static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue)
*/
#define STARTNODE (HUF_SYMBOLVALUE_MAX+1)
typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32];
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize)
{
nodeElt* const huffNode0 = (nodeElt*)workSpace;
nodeElt* const huffNode = huffNode0+1;
@@ -421,7 +421,7 @@ size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValu
* @return : maxNbBits
* Note : count is used before tree is written, so they can safely overlap
*/
size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits)
size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits)
{
huffNodeTable nodeTable;
return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable));
@@ -610,13 +610,14 @@ size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, si
return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0);
}
typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e;
static size_t HUF_compressCTable_internal(
BYTE* const ostart, BYTE* op, BYTE* const oend,
const void* src, size_t srcSize,
unsigned singleStream, const HUF_CElt* CTable, const int bmi2)
HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2)
{
size_t const cSize = singleStream ?
size_t const cSize = (nbStreams==HUF_singleStream) ?
HUF_compress1X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2) :
HUF_compress4X_usingCTable_internal(op, oend - op, src, srcSize, CTable, bmi2);
if (HUF_isError(cSize)) { return cSize; }
@@ -628,18 +629,18 @@ static size_t HUF_compressCTable_internal(
}
typedef struct {
U32 count[HUF_SYMBOLVALUE_MAX + 1];
unsigned count[HUF_SYMBOLVALUE_MAX + 1];
HUF_CElt CTable[HUF_SYMBOLVALUE_MAX + 1];
huffNodeTable nodeTable;
} HUF_compress_tables_t;
/* HUF_compress_internal() :
* `workSpace` must a table of at least HUF_WORKSPACE_SIZE_U32 unsigned */
static size_t HUF_compress_internal (
void* dst, size_t dstSize,
static size_t
HUF_compress_internal (void* dst, size_t dstSize,
const void* src, size_t srcSize,
unsigned maxSymbolValue, unsigned huffLog,
unsigned singleStream,
HUF_nbStreams_e nbStreams,
void* workSpace, size_t wkspSize,
HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat,
const int bmi2)
@@ -651,7 +652,7 @@ static size_t HUF_compress_internal (
/* checks & inits */
if (((size_t)workSpace & 3) != 0) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */
if (wkspSize < sizeof(*table)) return ERROR(workSpace_tooSmall);
if (wkspSize < HUF_WORKSPACE_SIZE) return ERROR(workSpace_tooSmall);
if (!srcSize) return 0; /* Uncompressed */
if (!dstSize) return 0; /* cannot fit anything within dst budget */
if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */
@@ -664,11 +665,11 @@ static size_t HUF_compress_internal (
if (preferRepeat && repeat && *repeat == HUF_repeat_valid) {
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, oldHufTable, bmi2);
nbStreams, oldHufTable, bmi2);
}
/* Scan input and build symbol stats */
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->count) );
{ CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize) );
if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */
if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */
}
@@ -683,14 +684,15 @@ static size_t HUF_compress_internal (
if (preferRepeat && repeat && *repeat != HUF_repeat_none) {
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, oldHufTable, bmi2);
nbStreams, oldHufTable, bmi2);
}
/* Build Huffman Tree */
huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue);
{ CHECK_V_F(maxBits, HUF_buildCTable_wksp(table->CTable, table->count,
{ size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count,
maxSymbolValue, huffLog,
table->nodeTable, sizeof(table->nodeTable)) );
table->nodeTable, sizeof(table->nodeTable));
CHECK_F(maxBits);
huffLog = (U32)maxBits;
/* Zero unused symbols in CTable, so we can check it for validity */
memset(table->CTable + (maxSymbolValue + 1), 0,
@@ -706,7 +708,7 @@ static size_t HUF_compress_internal (
if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) {
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, oldHufTable, bmi2);
nbStreams, oldHufTable, bmi2);
} }
/* Use the new huffman table */
@@ -718,7 +720,7 @@ static size_t HUF_compress_internal (
}
return HUF_compressCTable_internal(ostart, op, oend,
src, srcSize,
singleStream, table->CTable, bmi2);
nbStreams, table->CTable, bmi2);
}
@@ -728,7 +730,7 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize,
void* workSpace, size_t wkspSize)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 1 /*single stream*/,
maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize,
NULL, NULL, 0, 0 /*bmi2*/);
}
@@ -740,7 +742,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 1 /*single stream*/,
maxSymbolValue, huffLog, HUF_singleStream,
workSpace, wkspSize, hufTable,
repeat, preferRepeat, bmi2);
}
@@ -762,7 +764,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize,
void* workSpace, size_t wkspSize)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 0 /*4 streams*/,
maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize,
NULL, NULL, 0, 0 /*bmi2*/);
}
@@ -777,7 +779,7 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize,
HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2)
{
return HUF_compress_internal(dst, dstSize, src, srcSize,
maxSymbolValue, huffLog, 0 /* 4 streams */,
maxSymbolValue, huffLog, HUF_fourStreams,
workSpace, wkspSize,
hufTable, repeat, preferRepeat, bmi2);
}

220
C/zstd/huf_decompress.c
View File

@@ -43,6 +43,19 @@
#include "huf.h"
#include "error_private.h"
/* **************************************************************
* Macros
****************************************************************/
/* These two optional macros force the use one way or another of the two
* Huffman decompression implementations. You can't force in both directions
* at the same time.
*/
#if defined(HUF_FORCE_DECOMPRESS_X1) && \
defined(HUF_FORCE_DECOMPRESS_X2)
#error "Cannot force the use of the X1 and X2 decoders at the same time!"
#endif
/* **************************************************************
* Error Management
@@ -58,6 +71,51 @@
#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask))
/* **************************************************************
* BMI2 Variant Wrappers
****************************************************************/
#if DYNAMIC_BMI2
#define HUF_DGEN(fn) \
\
static size_t fn##_default( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
if (bmi2) { \
return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#else
#define HUF_DGEN(fn) \
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
(void)bmi2; \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#endif
/*-***************************/
/* generic DTableDesc */
/*-***************************/
@@ -71,6 +129,8 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table)
}
#ifndef HUF_FORCE_DECOMPRESS_X2
/*-***************************/
/* single-symbol decoding */
/*-***************************/
@@ -307,46 +367,6 @@ typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize,
const void *cSrc,
size_t cSrcSize,
const HUF_DTable *DTable);
#if DYNAMIC_BMI2
#define HUF_DGEN(fn) \
\
static size_t fn##_default( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \
void* dst, size_t dstSize, \
const void* cSrc, size_t cSrcSize, \
const HUF_DTable* DTable) \
{ \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
\
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
if (bmi2) { \
return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \
} \
return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#else
#define HUF_DGEN(fn) \
static size_t fn(void* dst, size_t dstSize, void const* cSrc, \
size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \
{ \
(void)bmi2; \
return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \
}
#endif
HUF_DGEN(HUF_decompress1X1_usingDTable_internal)
HUF_DGEN(HUF_decompress4X1_usingDTable_internal)
@@ -437,6 +457,10 @@ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cS
return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
#endif /* HUF_FORCE_DECOMPRESS_X2 */
#ifndef HUF_FORCE_DECOMPRESS_X1
/* *************************/
/* double-symbols decoding */
@@ -911,6 +935,8 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS
return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize);
}
#endif /* HUF_FORCE_DECOMPRESS_X1 */
/* ***********************************/
/* Universal decompression selectors */
@@ -921,8 +947,18 @@ size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#else
return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#endif
}
size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
@@ -930,11 +966,22 @@ size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize,
const HUF_DTable* DTable)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#else
return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) :
HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0);
#endif
}
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t;
static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] =
{
@@ -956,6 +1003,7 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu
{{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */
{{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */
};
#endif
/** HUF_selectDecoder() :
* Tells which decoder is likely to decode faster,
@@ -966,6 +1014,15 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
{
assert(dstSize > 0);
assert(dstSize <= 128*1024);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dstSize;
(void)cSrcSize;
return 0;
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dstSize;
(void)cSrcSize;
return 1;
#else
/* decoder timing evaluation */
{ U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */
U32 const D256 = (U32)(dstSize >> 8);
@@ -973,14 +1030,18 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize)
U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256);
DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */
return DTime1 < DTime0;
} }
}
#endif
}
typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize);
size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize)
{
#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2)
static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 };
#endif
/* validation checks */
if (dstSize == 0) return ERROR(dstSize_tooSmall);
@@ -989,7 +1050,17 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize);
#else
return decompress[algoNb](dst, dstSize, cSrc, cSrcSize);
#endif
}
}
@@ -1002,8 +1073,18 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize);
#else
return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) :
HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ;
#endif
}
}
@@ -1025,8 +1106,19 @@ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst,
if (cSrcSize == 0) return ERROR(corruption_detected);
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize):
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
#else
return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize):
HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize);
#endif
}
}
@@ -1041,10 +1133,22 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize,
if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize);
#else
return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize):
HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc,
cSrcSize, workSpace, wkspSize);
#endif
}
}
@@ -1060,10 +1164,21 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize,
size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#else
return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#endif
}
#ifndef HUF_FORCE_DECOMPRESS_X2
size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
{
const BYTE* ip = (const BYTE*) cSrc;
@@ -1075,12 +1190,23 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS
return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2);
}
#endif
size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2)
{
DTableDesc const dtd = HUF_getDTableDesc(DTable);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)dtd;
assert(dtd.tableType == 0);
return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)dtd;
assert(dtd.tableType == 1);
return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#else
return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) :
HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2);
#endif
}
size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2)
@@ -1090,7 +1216,17 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds
if (cSrcSize == 0) return ERROR(corruption_detected);
{ U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize);
#if defined(HUF_FORCE_DECOMPRESS_X1)
(void)algoNb;
assert(algoNb == 0);
return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
#elif defined(HUF_FORCE_DECOMPRESS_X2)
(void)algoNb;
assert(algoNb == 1);
return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
#else
return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) :
HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2);
#endif
}
}

2
C/zstd/pool.c
View File

@@ -88,8 +88,8 @@ static void* POOL_thread(void* opaque) {
ctx->numThreadsBusy++;
ctx->queueEmpty = ctx->queueHead == ctx->queueTail;
/* Unlock the mutex, signal a pusher, and run the job */
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
ZSTD_pthread_cond_signal(&ctx->queuePushCond);
ZSTD_pthread_mutex_unlock(&ctx->queueMutex);
job.function(job.opaque);

1502
C/zstd/zstd.h
View File

File diff suppressed because it is too large Load Diff

4
C/zstd/zstd_common.c
View File

@@ -30,8 +30,10 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; }
/*-****************************************
* ZSTD Error Management
******************************************/
#undef ZSTD_isError /* defined within zstd_internal.h */
/*! ZSTD_isError() :
* tells if a return value is an error code */
* tells if a return value is an error code
* symbol is required for external callers */
unsigned ZSTD_isError(size_t code) { return ERR_isError(code); }
/*! ZSTD_getErrorName() :

948
C/zstd/zstd_compress.c
View File

File diff suppressed because it is too large Load Diff

112
C/zstd/zstd_compress_internal.h
View File

@@ -48,12 +48,6 @@ extern "C" {
typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e;
typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage;
typedef enum {
ZSTD_dictDefaultAttach = 0,
ZSTD_dictForceAttach = 1,
ZSTD_dictForceCopy = -1,
} ZSTD_dictAttachPref_e;
typedef struct ZSTD_prefixDict_s {
const void* dict;
size_t dictSize;
@@ -96,10 +90,10 @@ typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e;
typedef struct {
/* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */
U32* litFreq; /* table of literals statistics, of size 256 */
U32* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
U32* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
U32* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
unsigned* litFreq; /* table of literals statistics, of size 256 */
unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */
unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */
unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */
ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */
ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */
@@ -167,7 +161,7 @@ typedef struct {
U32 hashLog; /* Log size of hashTable */
U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */
U32 minMatchLength; /* Minimum match length */
U32 hashEveryLog; /* Log number of entries to skip */
U32 hashRateLog; /* Log number of entries to skip */
U32 windowLog; /* Window log for the LDM */
} ldmParams_t;
@@ -196,9 +190,10 @@ struct ZSTD_CCtx_params_s {
ZSTD_dictAttachPref_e attachDictPref;
/* Multithreading: used to pass parameters to mtctx */
unsigned nbWorkers;
unsigned jobSize;
unsigned overlapSizeLog;
int nbWorkers;
size_t jobSize;
int overlapLog;
int rsyncable;
/* Long distance matching parameters */
ldmParams_t ldmParams;
@@ -498,6 +493,64 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls)
}
}
/** ZSTD_ipow() :
* Return base^exponent.
*/
static U64 ZSTD_ipow(U64 base, U64 exponent)
{
U64 power = 1;
while (exponent) {
if (exponent & 1) power *= base;
exponent >>= 1;
base *= base;
}
return power;
}
#define ZSTD_ROLL_HASH_CHAR_OFFSET 10
/** ZSTD_rollingHash_append() :
* Add the buffer to the hash value.
*/
static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size)
{
BYTE const* istart = (BYTE const*)buf;
size_t pos;
for (pos = 0; pos < size; ++pos) {
hash *= prime8bytes;
hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET;
}
return hash;
}
/** ZSTD_rollingHash_compute() :
* Compute the rolling hash value of the buffer.
*/
MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size)
{
return ZSTD_rollingHash_append(0, buf, size);
}
/** ZSTD_rollingHash_primePower() :
* Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash
* over a window of length bytes.
*/
MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length)
{
return ZSTD_ipow(prime8bytes, length - 1);
}
/** ZSTD_rollingHash_rotate() :
* Rotate the rolling hash by one byte.
*/
MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower)
{
hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower;
hash *= prime8bytes;
hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET;
return hash;
}
/*-*************************************
* Round buffer management
***************************************/
@@ -626,20 +679,23 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog,
* dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary
* is below them. forceWindow and dictMatchState are therefore incompatible.
*/
MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
void const* srcEnd, U32 maxDist,
MEM_STATIC void
ZSTD_window_enforceMaxDist(ZSTD_window_t* window,
void const* srcEnd,
U32 maxDist,
U32* loadedDictEndPtr,
const ZSTD_matchState_t** dictMatchStatePtr)
{
U32 const current = (U32)((BYTE const*)srcEnd - window->base);
U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0;
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: current=%u, maxDist=%u", current, maxDist);
if (current > maxDist + loadedDictEnd) {
U32 const newLowLimit = current - maxDist;
U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base);
U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0;
DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u",
(unsigned)blockEndIdx, (unsigned)maxDist);
if (blockEndIdx > maxDist + loadedDictEnd) {
U32 const newLowLimit = blockEndIdx - maxDist;
if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit;
if (window->dictLimit < window->lowLimit) {
DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u",
window->dictLimit, window->lowLimit);
(unsigned)window->dictLimit, (unsigned)window->lowLimit);
window->dictLimit = window->lowLimit;
}
if (loadedDictEndPtr)
@@ -690,20 +746,23 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window,
/* debug functions */
#if (DEBUGLEVEL>=2)
MEM_STATIC double ZSTD_fWeight(U32 rawStat)
{
U32 const fp_accuracy = 8;
U32 const fp_multiplier = (1 << fp_accuracy);
U32 const stat = rawStat + 1;
U32 const hb = ZSTD_highbit32(stat);
U32 const newStat = rawStat + 1;
U32 const hb = ZSTD_highbit32(newStat);
U32 const BWeight = hb * fp_multiplier;
U32 const FWeight = (stat << fp_accuracy) >> hb;
U32 const FWeight = (newStat << fp_accuracy) >> hb;
U32 const weight = BWeight + FWeight;
assert(hb + fp_accuracy < 31);
return (double)weight / fp_multiplier;
}
/* display a table content,
* listing each element, its frequency, and its predicted bit cost */
MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
{
unsigned u, sum;
@@ -715,6 +774,9 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max)
}
}
#endif
#if defined (__cplusplus)
}
#endif

240
C/zstd/zstd_ddict.c Normal file
View File

@@ -0,0 +1,240 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_ddict.c :
* concentrates all logic that needs to know the internals of ZSTD_DDict object */
/*-*******************************************************
* Dependencies
*********************************************************/
#include <string.h> /* memcpy, memmove, memset */
#include "cpu.h" /* bmi2 */
#include "mem.h" /* low level memory routines */
#define FSE_STATIC_LINKING_ONLY
#include "fse.h"
#define HUF_STATIC_LINKING_ONLY
#include "huf.h"
#include "zstd_decompress_internal.h"
#include "zstd_ddict.h"
#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1)
# include "zstd_legacy.h"
#endif
/*-*******************************************************
* Types
*********************************************************/
struct ZSTD_DDict_s {
void* dictBuffer;
const void* dictContent;
size_t dictSize;
ZSTD_entropyDTables_t entropy;
U32 dictID;
U32 entropyPresent;
ZSTD_customMem cMem;
}; /* typedef'd to ZSTD_DDict within "zstd.h" */
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictContent;
}
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict)
{
assert(ddict != NULL);
return ddict->dictSize;
}
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict)
{
DEBUGLOG(4, "ZSTD_copyDDictParameters");
assert(dctx != NULL);
assert(ddict != NULL);
dctx->dictID = ddict->dictID;
dctx->prefixStart = ddict->dictContent;
dctx->virtualStart = ddict->dictContent;
dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize;
dctx->previousDstEnd = dctx->dictEnd;
if (ddict->entropyPresent) {
dctx->litEntropy = 1;
dctx->fseEntropy = 1;
dctx->LLTptr = ddict->entropy.LLTable;
dctx->MLTptr = ddict->entropy.MLTable;
dctx->OFTptr = ddict->entropy.OFTable;
dctx->HUFptr = ddict->entropy.hufTable;
dctx->entropy.rep[0] = ddict->entropy.rep[0];
dctx->entropy.rep[1] = ddict->entropy.rep[1];
dctx->entropy.rep[2] = ddict->entropy.rep[2];
} else {
dctx->litEntropy = 0;
dctx->fseEntropy = 0;
}
}
static size_t
ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict,
ZSTD_dictContentType_e dictContentType)
{
ddict->dictID = 0;
ddict->entropyPresent = 0;
if (dictContentType == ZSTD_dct_rawContent) return 0;
if (ddict->dictSize < 8) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
{ U32 const magic = MEM_readLE32(ddict->dictContent);
if (magic != ZSTD_MAGIC_DICTIONARY) {
if (dictContentType == ZSTD_dct_fullDict)
return ERROR(dictionary_corrupted); /* only accept specified dictionaries */
return 0; /* pure content mode */
}
}
ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE);
/* load entropy tables */
CHECK_E( ZSTD_loadDEntropy(&ddict->entropy,
ddict->dictContent, ddict->dictSize),
dictionary_corrupted );
ddict->entropyPresent = 1;
return 0;
}
static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) {
ddict->dictBuffer = NULL;
ddict->dictContent = dict;
if (!dict) dictSize = 0;
} else {
void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem);
ddict->dictBuffer = internalBuffer;
ddict->dictContent = internalBuffer;
if (!internalBuffer) return ERROR(memory_allocation);
memcpy(internalBuffer, dict, dictSize);
}
ddict->dictSize = dictSize;
ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */
/* parse dictionary content */
CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) );
return 0;
}
ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType,
ZSTD_customMem customMem)
{
if (!customMem.customAlloc ^ !customMem.customFree) return NULL;
{ ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem);
if (ddict == NULL) return NULL;
ddict->cMem = customMem;
{ size_t const initResult = ZSTD_initDDict_internal(ddict,
dict, dictSize,
dictLoadMethod, dictContentType);
if (ZSTD_isError(initResult)) {
ZSTD_freeDDict(ddict);
return NULL;
} }
return ddict;
}
}
/*! ZSTD_createDDict() :
* Create a digested dictionary, to start decompression without startup delay.
* `dict` content is copied inside DDict.
* Consequently, `dict` can be released after `ZSTD_DDict` creation */
ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator);
}
/*! ZSTD_createDDict_byReference() :
* Create a digested dictionary, to start decompression without startup delay.
* Dictionary content is simply referenced, it will be accessed during decompression.
* Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */
ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize)
{
ZSTD_customMem const allocator = { NULL, NULL, NULL };
return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator);
}
const ZSTD_DDict* ZSTD_initStaticDDict(
void* sBuffer, size_t sBufferSize,
const void* dict, size_t dictSize,
ZSTD_dictLoadMethod_e dictLoadMethod,
ZSTD_dictContentType_e dictContentType)
{
size_t const neededSpace = sizeof(ZSTD_DDict)
+ (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer;
assert(sBuffer != NULL);
assert(dict != NULL);
if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */
if (sBufferSize < neededSpace) return NULL;
if (dictLoadMethod == ZSTD_dlm_byCopy) {
memcpy(ddict+1, dict, dictSize); /* local copy */
dict = ddict+1;
}
if (ZSTD_isError( ZSTD_initDDict_internal(ddict,
dict, dictSize,
ZSTD_dlm_byRef, dictContentType) ))
return NULL;
return ddict;
}
size_t ZSTD_freeDDict(ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support free on NULL */
{ ZSTD_customMem const cMem = ddict->cMem;
ZSTD_free(ddict->dictBuffer, cMem);
ZSTD_free(ddict, cMem);
return 0;
}
}
/*! ZSTD_estimateDDictSize() :
* Estimate amount of memory that will be needed to create a dictionary for decompression.
* Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */
size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod)
{
return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize);
}
size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0; /* support sizeof on NULL */
return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ;
}
/*! ZSTD_getDictID_fromDDict() :
* Provides the dictID of the dictionary loaded into `ddict`.
* If @return == 0, the dictionary is not conformant to Zstandard specification, or empty.
* Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */
unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict)
{
if (ddict==NULL) return 0;
return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize);
}

44
C/zstd/zstd_ddict.h Normal file
View File

@@ -0,0 +1,44 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_DDICT_H
#define ZSTD_DDICT_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include <stddef.h> /* size_t */
#include "zstd.h" /* ZSTD_DDict, and several public functions */
/*-*******************************************************
* Interface
*********************************************************/
/* note: several prototypes are already published in `zstd.h` :
* ZSTD_createDDict()
* ZSTD_createDDict_byReference()
* ZSTD_createDDict_advanced()
* ZSTD_freeDDict()
* ZSTD_initStaticDDict()
* ZSTD_sizeof_DDict()
* ZSTD_estimateDDictSize()
* ZSTD_getDictID_fromDict()
*/
const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict);
size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict);
void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict);
#endif /* ZSTD_DDICT_H */

1912
C/zstd/zstd_decompress.c
View File

File diff suppressed because it is too large Load Diff

1307
C/zstd/zstd_decompress_block.c Normal file
View File

File diff suppressed because it is too large Load Diff

59
C/zstd/zstd_decompress_block.h Normal file
View File

@@ -0,0 +1,59 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
#ifndef ZSTD_DEC_BLOCK_H
#define ZSTD_DEC_BLOCK_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include <stddef.h> /* size_t */
#include "zstd.h" /* DCtx, and some public functions */
#include "zstd_internal.h" /* blockProperties_t, and some public functions */
#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */
/* === Prototypes === */
/* note: prototypes already published within `zstd.h` :
* ZSTD_decompressBlock()
*/
/* note: prototypes already published within `zstd_internal.h` :
* ZSTD_getcBlockSize()
* ZSTD_decodeSeqHeaders()
*/
/* ZSTD_decompressBlock_internal() :
* decompress block, starting at `src`,
* into destination buffer `dst`.
* @return : decompressed block size,
* or an error code (which can be tested using ZSTD_isError())
*/
size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize, const int frame);
/* ZSTD_buildFSETable() :
* generate FSE decoding table for one symbol (ll, ml or off)
* this function must be called with valid parameters only
* (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.)
* in which case it cannot fail.
* Internal use only.
*/
void ZSTD_buildFSETable(ZSTD_seqSymbol* dt,
const short* normalizedCounter, unsigned maxSymbolValue,
const U32* baseValue, const U32* nbAdditionalBits,
unsigned tableLog);
#endif /* ZSTD_DEC_BLOCK_H */

168
C/zstd/zstd_decompress_internal.h Normal file
View File

@@ -0,0 +1,168 @@
/*
* Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under both the BSD-style license (found in the
* LICENSE file in the root directory of this source tree) and the GPLv2 (found
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
/* zstd_decompress_internal:
* objects and definitions shared within lib/decompress modules */
#ifndef ZSTD_DECOMPRESS_INTERNAL_H
#define ZSTD_DECOMPRESS_INTERNAL_H
/*-*******************************************************
* Dependencies
*********************************************************/
#include "mem.h" /* BYTE, U16, U32 */
#include "zstd_internal.h" /* ZSTD_seqSymbol */
/*-*******************************************************
* Constants
*********************************************************/
static const U32 LL_base[MaxLL+1] = {
0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 18, 20, 22, 24, 28, 32, 40,
48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000,
0x2000, 0x4000, 0x8000, 0x10000 };
static const U32 OF_base[MaxOff+1] = {
0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D,
0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD,
0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD,
0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD };
static const U32 OF_bits[MaxOff+1] = {
0, 1, 2, 3, 4, 5, 6, 7,
8, 9, 10, 11, 12, 13, 14, 15,
16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31 };
static const U32 ML_base[MaxML+1] = {
3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18,
19, 20, 21, 22, 23, 24, 25, 26,
27, 28, 29, 30, 31, 32, 33, 34,
35, 37, 39, 41, 43, 47, 51, 59,
67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803,
0x1003, 0x2003, 0x4003, 0x8003, 0x10003 };
/*-*******************************************************
* Decompression types
*********************************************************/
typedef struct {
U32 fastMode;
U32 tableLog;
} ZSTD_seqSymbol_header;
typedef struct {
U16 nextState;
BYTE nbAdditionalBits;
BYTE nbBits;
U32 baseValue;
} ZSTD_seqSymbol;
#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log)))
typedef struct {
ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */
ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */
ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */
HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */
U32 rep[ZSTD_REP_NUM];
} ZSTD_entropyDTables_t;
typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader,
ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock,
ZSTDds_decompressLastBlock, ZSTDds_checkChecksum,
ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage;
typedef enum { zdss_init=0, zdss_loadHeader,
zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage;
struct ZSTD_DCtx_s
{
const ZSTD_seqSymbol* LLTptr;
const ZSTD_seqSymbol* MLTptr;
const ZSTD_seqSymbol* OFTptr;
const HUF_DTable* HUFptr;
ZSTD_entropyDTables_t entropy;
U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */
const void* previousDstEnd; /* detect continuity */
const void* prefixStart; /* start of current segment */
const void* virtualStart; /* virtual start of previous segment if it was just before current one */
const void* dictEnd; /* end of previous segment */
size_t expected;
ZSTD_frameHeader fParams;
U64 decodedSize;
blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */
ZSTD_dStage stage;
U32 litEntropy;
U32 fseEntropy;
XXH64_state_t xxhState;
size_t headerSize;
ZSTD_format_e format;
const BYTE* litPtr;
ZSTD_customMem customMem;
size_t litSize;
size_t rleSize;
size_t staticSize;
int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */
/* dictionary */
ZSTD_DDict* ddictLocal;
const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */
U32 dictID;
int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */
/* streaming */
ZSTD_dStreamStage streamStage;
char* inBuff;
size_t inBuffSize;
size_t inPos;
size_t maxWindowSize;
char* outBuff;
size_t outBuffSize;
size_t outStart;
size_t outEnd;
size_t lhSize;
void* legacyContext;
U32 previousLegacyVersion;
U32 legacyVersion;
U32 hostageByte;
int noForwardProgress;
/* workspace */
BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH];
BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX];
}; /* typedef'd to ZSTD_DCtx within "zstd.h" */
/*-*******************************************************
* Shared internal functions
*********************************************************/
/*! ZSTD_loadDEntropy() :
* dict : must point at beginning of a valid zstd dictionary.
* @return : size of entropy tables read */
size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy,
const void* const dict, size_t const dictSize);
/*! ZSTD_checkContinuity() :
* check if next `dst` follows previous position, where decompression ended.
* If yes, do nothing (continue on current segment).
* If not, classify previous segment as "external dictionary", and start a new segment.
* This function cannot fail. */
void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst);
#endif /* ZSTD_DECOMPRESS_INTERNAL_H */

8
C/zstd/zstd_double_fast.c
View File

@@ -18,7 +18,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashLarge = ms->hashTable;
U32 const hBitsL = cParams->hashLog;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
U32* const hashSmall = ms->chainTable;
U32 const hBitsS = cParams->chainLog;
const BYTE* const base = ms->window.base;
@@ -309,7 +309,7 @@ size_t ZSTD_compressBlock_doubleFast(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.searchLength;
const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
@@ -329,7 +329,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
const U32 mls = ms->cParams.searchLength;
const U32 mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */
@@ -483,7 +483,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize)
{
U32 const mls = ms->cParams.searchLength;
U32 const mls = ms->cParams.minMatch;
switch(mls)
{
default: /* includes case 3 */

1
C/zstd/zstd_errors.h
View File

@@ -72,6 +72,7 @@ typedef enum {
ZSTD_error_workSpace_tooSmall= 66,
ZSTD_error_dstSize_tooSmall = 70,
ZSTD_error_srcSize_wrong = 72,
ZSTD_error_dstBuffer_null = 74,
/* following error codes are __NOT STABLE__, they can be removed or changed in future versions */
ZSTD_error_frameIndex_tooLarge = 100,
ZSTD_error_seekableIO = 102,

28
C/zstd/zstd_fast.c
View File

@@ -18,7 +18,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const hashTable = ms->hashTable;
U32 const hBits = cParams->hashLog;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
const BYTE* const base = ms->window.base;
const BYTE* ip = base + ms->nextToUpdate;
const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE;
@@ -27,18 +27,18 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms,
/* Always insert every fastHashFillStep position into the hash table.
* Insert the other positions if their hash entry is empty.
*/
for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) {
for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) {
U32 const current = (U32)(ip - base);
U32 i;
for (i = 0; i < fastHashFillStep; ++i) {
size_t const hash = ZSTD_hashPtr(ip + i, hBits, mls);
if (i == 0 || hashTable[hash] == 0)
hashTable[hash] = current + i;
size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls);
hashTable[hash0] = current;
if (dtlm == ZSTD_dtlm_fast) continue;
/* Only load extra positions for ZSTD_dtlm_full */
if (dtlm == ZSTD_dtlm_fast)
break;
}
}
{ U32 p;
for (p = 1; p < fastHashFillStep; ++p) {
size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls);
if (hashTable[hash] == 0) { /* not yet filled */
hashTable[hash] = current + p;
} } } }
}
FORCE_INLINE_TEMPLATE
@@ -235,7 +235,7 @@ size_t ZSTD_compressBlock_fast(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
assert(ms->dictMatchState == NULL);
switch(mls)
{
@@ -256,7 +256,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
assert(ms->dictMatchState != NULL);
switch(mls)
{
@@ -375,7 +375,7 @@ size_t ZSTD_compressBlock_fast_extDict(
void const* src, size_t srcSize)
{
ZSTD_compressionParameters const* cParams = &ms->cParams;
U32 const mls = cParams->searchLength;
U32 const mls = cParams->minMatch;
switch(mls)
{
default: /* includes case 3 */

13
C/zstd/zstd_internal.h
View File

@@ -41,6 +41,9 @@ extern "C" {
/* ---- static assert (debug) --- */
#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c)
#define ZSTD_isError ERR_isError /* for inlining */
#define FSE_isError ERR_isError
#define HUF_isError ERR_isError
/*-*************************************
@@ -75,7 +78,6 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 };
#define BIT0 1
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10
#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */
static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 };
static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 };
@@ -242,7 +244,7 @@ typedef struct {
blockType_e blockType;
U32 lastBlock;
U32 origSize;
} blockProperties_t;
} blockProperties_t; /* declared here for decompress and fullbench */
/*! ZSTD_getcBlockSize() :
* Provides the size of compressed block from block header `src` */
@@ -250,6 +252,13 @@ typedef struct {
size_t ZSTD_getcBlockSize(const void* src, size_t srcSize,
blockProperties_t* bpPtr);
/*! ZSTD_decodeSeqHeaders() :
* decode sequence header from src */
/* Used by: decompress, fullbench (does not get its definition from here) */
size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr,
const void* src, size_t srcSize);
#if defined (__cplusplus)
}
#endif

33
C/zstd/zstd_lazy.c
View File

@@ -63,7 +63,8 @@ ZSTD_updateDUBT(ZSTD_matchState_t* ms,
static void
ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
U32 current, const BYTE* inputEnd,
U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode)
U32 nbCompares, U32 btLow,
const ZSTD_dictMode_e dictMode)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32* const bt = ms->chainTable;
@@ -80,7 +81,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
const BYTE* match;
U32* smallerPtr = bt + 2*(current&btMask);
U32* largerPtr = smallerPtr + 1;
U32 matchIndex = *smallerPtr;
U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = ms->window.lowLimit;
@@ -93,6 +94,9 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < current);
/* note : all candidates are now supposed sorted,
* but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK
* when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */
if ( (dictMode != ZSTD_extDict)
|| (matchIndex+matchLength >= dictLimit) /* both in current segment*/
@@ -108,7 +112,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms,
match = dictBase + matchIndex;
matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart);
if (matchIndex+matchLength >= dictLimit)
match = base + matchIndex; /* to prepare for next usage of match[matchLength] */
match = base + matchIndex; /* preparation for next read of match[matchLength] */
}
DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ",
@@ -258,7 +262,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
&& (nbCandidates > 1) ) {
DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted",
matchIndex);
*unsortedMark = previousCandidate;
*unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */
previousCandidate = matchIndex;
matchIndex = *nextCandidate;
nextCandidate = bt + 2*(matchIndex&btMask);
@@ -266,11 +270,13 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms,
nbCandidates --;
}
/* nullify last candidate if it's still unsorted
* simplification, detrimental to compression ratio, beneficial for speed */
if ( (matchIndex > unsortLimit)
&& (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) {
DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u",
matchIndex);
*nextCandidate = *unsortedMark = 0; /* nullify next candidate if it's still unsorted (note : simplification, detrimental to compression ratio, beneficial for speed) */
*nextCandidate = *unsortedMark = 0;
}
/* batch sort stacked candidates */
@@ -386,7 +392,7 @@ ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms,
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
@@ -402,7 +408,7 @@ static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
@@ -418,7 +424,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
@@ -433,7 +439,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
/* *********************************
* Hash Chain
***********************************/
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask]
#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)]
/* Update chains up to ip (excluded)
Assumption : always within prefix (i.e. not within extDict) */
@@ -463,7 +469,7 @@ static U32 ZSTD_insertAndFindFirstIndex_internal(
U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) {
const ZSTD_compressionParameters* const cParams = &ms->cParams;
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.searchLength);
return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch);
}
@@ -497,6 +503,7 @@ size_t ZSTD_HcFindBestMatch_generic (
size_t currentMl=0;
if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) {
const BYTE* const match = base + matchIndex;
assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */
if (match[ml] == ip[ml]) /* potentially better */
currentMl = ZSTD_count(ip, match, iLimit);
} else {
@@ -559,7 +566,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
@@ -575,7 +582,7 @@ static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
@@ -591,7 +598,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
const BYTE* ip, const BYTE* const iLimit,
size_t* offsetPtr)
{
switch(ms->cParams.searchLength)
switch(ms->cParams.minMatch)
{
default : /* includes case 3 */
case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);

77
C/zstd/zstd_ldm.c
View File

@@ -37,8 +37,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params,
params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG);
assert(params->hashLog <= ZSTD_HASHLOG_MAX);
}
if (params->hashEveryLog == 0) {
params->hashEveryLog = params->windowLog < params->hashLog
if (params->hashRateLog == 0) {
params->hashRateLog = params->windowLog < params->hashLog
? 0
: params->windowLog - params->hashLog;
}
@@ -119,20 +119,20 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState,
*
* Gets the small hash, checksum, and tag from the rollingHash.
*
* If the tag matches (1 << ldmParams.hashEveryLog)-1, then
* If the tag matches (1 << ldmParams.hashRateLog)-1, then
* creates an ldmEntry from the offset, and inserts it into the hash table.
*
* hBits is the length of the small hash, which is the most significant hBits
* of rollingHash. The checksum is the next 32 most significant bits, followed
* by ldmParams.hashEveryLog bits that make up the tag. */
* by ldmParams.hashRateLog bits that make up the tag. */
static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
U64 const rollingHash,
U32 const hBits,
U32 const offset,
ldmParams_t const ldmParams)
{
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog);
U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1;
U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog);
U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1;
if (tag == tagMask) {
U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits);
U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits);
@@ -143,56 +143,6 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState,
}
}
/** ZSTD_ldm_getRollingHash() :
* Get a 64-bit hash using the first len bytes from buf.
*
* Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be
* H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0)
*
* where the constant a is defined to be prime8bytes.
*
* The implementation adds an offset to each byte, so
* H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */
static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len)
{
U64 ret = 0;
U32 i;
for (i = 0; i < len; i++) {
ret *= prime8bytes;
ret += buf[i] + LDM_HASH_CHAR_OFFSET;
}
return ret;
}
/** ZSTD_ldm_ipow() :
* Return base^exp. */
static U64 ZSTD_ldm_ipow(U64 base, U64 exp)
{
U64 ret = 1;
while (exp) {
if (exp & 1) { ret *= base; }
exp >>= 1;
base *= base;
}
return ret;
}
U64 ZSTD_ldm_getHashPower(U32 minMatchLength) {
DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength);
assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN);
return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1);
}
/** ZSTD_ldm_updateHash() :
* Updates hash by removing toRemove and adding toAdd. */
static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower)
{
hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower);
hash *= prime8bytes;
hash += toAdd + LDM_HASH_CHAR_OFFSET;
return hash;
}
/** ZSTD_ldm_countBackwardsMatch() :
* Returns the number of bytes that match backwards before pIn and pMatch.
*
@@ -238,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms,
case ZSTD_btlazy2:
case ZSTD_btopt:
case ZSTD_btultra:
case ZSTD_btultra2:
break;
default:
assert(0); /* not possible : not a valid strategy id */
@@ -261,7 +212,7 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state,
const BYTE* cur = lastHashed + 1;
while (cur < iend) {
rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1],
rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1],
cur[ldmParams.minMatchLength-1],
state->hashPower);
ZSTD_ldm_makeEntryAndInsertByTag(state,
@@ -297,8 +248,8 @@ static size_t ZSTD_ldm_generateSequences_internal(
U64 const hashPower = ldmState->hashPower;
U32 const hBits = params->hashLog - params->bucketSizeLog;
U32 const ldmBucketSize = 1U << params->bucketSizeLog;
U32 const hashEveryLog = params->hashEveryLog;
U32 const ldmTagMask = (1U << params->hashEveryLog) - 1;
U32 const hashRateLog = params->hashRateLog;
U32 const ldmTagMask = (1U << params->hashRateLog) - 1;
/* Prefix and extDict parameters */
U32 const dictLimit = ldmState->window.dictLimit;
U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit;
@@ -324,16 +275,16 @@ static size_t ZSTD_ldm_generateSequences_internal(
size_t forwardMatchLength = 0, backwardMatchLength = 0;
ldmEntry_t* bestEntry = NULL;
if (ip != istart) {
rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0],
rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0],
lastHashed[minMatchLength],
hashPower);
} else {
rollingHash = ZSTD_ldm_getRollingHash(ip, minMatchLength);
rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength);
}
lastHashed = ip;
/* Do not insert and do not look for a match */
if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) {
if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) {
ip++;
continue;
}
@@ -593,7 +544,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
void const* src, size_t srcSize)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
unsigned const minMatch = cParams->searchLength;
unsigned const minMatch = cParams->minMatch;
ZSTD_blockCompressor const blockCompressor =
ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms));
/* Input bounds */

8
C/zstd/zstd_ldm.h
View File

@@ -21,7 +21,7 @@ extern "C" {
* Long distance matching
***************************************/
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX
#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT
/**
* ZSTD_ldm_generateSequences():
@@ -86,12 +86,8 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params);
*/
size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize);
/** ZSTD_ldm_getTableSize() :
* Return prime8bytes^(minMatchLength-1) */
U64 ZSTD_ldm_getHashPower(U32 minMatchLength);
/** ZSTD_ldm_adjustParameters() :
* If the params->hashEveryLog is not set, set it to its default value based on
* If the params->hashRateLog is not set, set it to its default value based on
* windowLog and params->hashLog.
*
* Ensures that params->bucketSizeLog is <= params->hashLog (setting it to

173
C/zstd/zstd_opt.c
View File

@@ -17,6 +17,8 @@
#define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */
#define ZSTD_MAX_PRICE (1<<30)
#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */
/*-*************************************
* Price functions for optimal parser
@@ -52,11 +54,15 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat)
return weight;
}
/* debugging function, @return price in bytes */
#if (DEBUGLEVEL>=2)
/* debugging function,
* @return price in bytes as fractional value
* for debug messages only */
MEM_STATIC double ZSTD_fCost(U32 price)
{
return (double)price / (BITCOST_MULTIPLIER*8);
}
#endif
static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
{
@@ -67,29 +73,44 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel)
}
static U32 ZSTD_downscaleStat(U32* table, U32 lastEltIndex, int malus)
/* ZSTD_downscaleStat() :
* reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus)
* return the resulting sum of elements */
static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus)
{
U32 s, sum=0;
DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1);
assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31);
for (s=0; s<=lastEltIndex; s++) {
for (s=0; s<lastEltIndex+1; s++) {
table[s] = 1 + (table[s] >> (ZSTD_FREQ_DIV+malus));
sum += table[s];
}
return sum;
}
static void ZSTD_rescaleFreqs(optState_t* const optPtr,
/* ZSTD_rescaleFreqs() :
* if first block (detected by optPtr->litLengthSum == 0) : init statistics
* take hints from dictionary if there is one
* or init from zero, using src for literals stats, or flat 1 for match symbols
* otherwise downscale existing stats, to be used as seed for next block.
*/
static void
ZSTD_rescaleFreqs(optState_t* const optPtr,
const BYTE* const src, size_t const srcSize,
int optLevel)
int const optLevel)
{
DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize);
optPtr->priceType = zop_dynamic;
if (optPtr->litLengthSum == 0) { /* first block : init */
if (srcSize <= 1024) /* heuristic */
if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */
DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef");
optPtr->priceType = zop_predef;
}
assert(optPtr->symbolCosts != NULL);
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */
if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) {
/* huffman table presumed generated by dictionary */
optPtr->priceType = zop_dynamic;
assert(optPtr->litFreq != NULL);
@@ -208,7 +229,9 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP
/* dynamic statistics */
{ U32 const llCode = ZSTD_LLcode(litLength);
return (LL_bits[llCode] * BITCOST_MULTIPLIER) + (optPtr->litLengthSumBasePrice - WEIGHT(optPtr->litLengthFreq[llCode], optLevel));
return (LL_bits[llCode] * BITCOST_MULTIPLIER)
+ optPtr->litLengthSumBasePrice
- WEIGHT(optPtr->litLengthFreq[llCode], optLevel);
}
}
@@ -385,7 +408,6 @@ static U32 ZSTD_insertBt1(
U32* largerPtr = smallerPtr + 1;
U32 dummy32; /* to be nullified at the end */
U32 const windowLow = ms->window.lowLimit;
U32 const matchLow = windowLow ? windowLow : 1;
U32 matchEndIdx = current+8+1;
size_t bestLength = 8;
U32 nbCompares = 1U << cParams->searchLog;
@@ -401,7 +423,8 @@ static U32 ZSTD_insertBt1(
assert(ip <= iend-8); /* required for h calculation */
hashTable[h] = current; /* Update Hash Table */
while (nbCompares-- && (matchIndex >= matchLow)) {
assert(windowLow > 0);
while (nbCompares-- && (matchIndex >= windowLow)) {
U32* const nextPtr = bt + 2*(matchIndex & btMask);
size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */
assert(matchIndex < current);
@@ -479,7 +502,7 @@ void ZSTD_updateTree_internal(
const BYTE* const base = ms->window.base;
U32 const target = (U32)(ip - base);
U32 idx = ms->nextToUpdate;
DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)",
idx, target, dictMode);
while(idx < target)
@@ -488,15 +511,18 @@ void ZSTD_updateTree_internal(
}
void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) {
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict);
ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict);
}
FORCE_INLINE_TEMPLATE
U32 ZSTD_insertBtAndGetAllMatches (
ZSTD_matchState_t* ms,
const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode,
U32 rep[ZSTD_REP_NUM], U32 const ll0,
ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */)
U32 rep[ZSTD_REP_NUM],
U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */
ZSTD_match_t* matches,
const U32 lengthToBeat,
U32 const mls /* template */)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
@@ -542,6 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches (
DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current);
/* check repCode */
assert(ll0 <= 1); /* necessarily 1 or 0 */
{ U32 const lastR = ZSTD_REP_NUM + ll0;
U32 repCode;
for (repCode = ll0; repCode < lastR; repCode++) {
@@ -724,7 +751,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
ZSTD_match_t* matches, U32 const lengthToBeat)
{
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const matchLengthSearch = cParams->searchLength;
U32 const matchLengthSearch = cParams->minMatch;
DEBUGLOG(8, "ZSTD_BtGetAllMatches");
if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */
ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode);
@@ -774,12 +801,30 @@ static U32 ZSTD_totalLen(ZSTD_optimal_t sol)
return sol.litlen + sol.mlen;
}
#if 0 /* debug */
static void
listStats(const U32* table, int lastEltID)
{
int const nbElts = lastEltID + 1;
int enb;
for (enb=0; enb < nbElts; enb++) {
(void)table;
//RAWLOG(2, "%3i:%3i, ", enb, table[enb]);
RAWLOG(2, "%4i,", table[enb]);
}
RAWLOG(2, " \n");
}
#endif
FORCE_INLINE_TEMPLATE size_t
ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize,
const int optLevel, const ZSTD_dictMode_e dictMode)
const int optLevel,
const ZSTD_dictMode_e dictMode)
{
optState_t* const optStatePtr = &ms->opt;
const BYTE* const istart = (const BYTE*)src;
@@ -792,14 +837,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
const ZSTD_compressionParameters* const cParams = &ms->cParams;
U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4;
U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
ZSTD_optimal_t* const opt = optStatePtr->priceTable;
ZSTD_match_t* const matches = optStatePtr->matchTable;
ZSTD_optimal_t lastSequence;
/* init */
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic");
DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
(U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
assert(optLevel <= 2);
ms->nextToUpdate3 = ms->nextToUpdate;
ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
@@ -999,7 +1045,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */
U32 const offCode = opt[storePos].off;
U32 const advance = llen + mlen;
DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u",
anchor - istart, llen, mlen);
anchor - istart, (unsigned)llen, (unsigned)mlen);
if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */
assert(storePos == storeEnd); /* must be last sequence */
@@ -1047,11 +1093,11 @@ size_t ZSTD_compressBlock_btopt(
/* used in 2-pass strategy */
static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus)
{
U32 s, sum=0;
assert(ZSTD_FREQ_DIV+bonus > 0);
for (s=0; s<=lastEltIndex; s++) {
assert(ZSTD_FREQ_DIV+bonus >= 0);
for (s=0; s<lastEltIndex+1; s++) {
table[s] <<= ZSTD_FREQ_DIV+bonus;
table[s]--;
sum += table[s];
@@ -1063,9 +1109,43 @@ static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus)
MEM_STATIC void ZSTD_upscaleStats(optState_t* optPtr)
{
optPtr->litSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0);
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 1);
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1);
optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0);
optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0);
optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0);
}
/* ZSTD_initStats_ultra():
* make a first compression pass, just to seed stats with more accurate starting values.
* only works on first block, with no dictionary and no ldm.
* this function cannot error, hence its constract must be respected.
*/
static void
ZSTD_initStats_ultra(ZSTD_matchState_t* ms,
seqStore_t* seqStore,
U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */
memcpy(tmpRep, rep, sizeof(tmpRep));
DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize);
assert(ms->opt.litLengthSum == 0); /* first block */
assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */
assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */
assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
/* invalidate first scan from history */
ZSTD_resetSeqStore(seqStore);
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
}
size_t ZSTD_compressBlock_btultra(
@@ -1073,33 +1153,34 @@ size_t ZSTD_compressBlock_btultra(
const void* src, size_t srcSize)
{
DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize);
#if 0
/* 2-pass strategy (disabled)
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
const void* src, size_t srcSize)
{
U32 const current = (U32)((const BYTE*)src - ms->window.base);
DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize);
/* 2-pass strategy:
* this strategy makes a first pass over first block to collect statistics
* and seed next round's statistics with it.
* After 1st pass, function forgets everything, and starts a new block.
* Consequently, this can only work if no data has been previously loaded in tables,
* aka, no dictionary, no prefix, no ldm preprocessing.
* The compression ratio gain is generally small (~0.5% on first block),
* the cost is 2x cpu time on first block. */
assert(srcSize <= ZSTD_BLOCKSIZE_MAX);
if ( (ms->opt.litLengthSum==0) /* first block */
&& (seqStore->sequences == seqStore->sequencesStart) /* no ldm */
&& (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */
U32 tmpRep[ZSTD_REP_NUM];
DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics");
assert(ms->nextToUpdate >= ms->window.dictLimit
&& ms->nextToUpdate <= ms->window.dictLimit + 1);
memcpy(tmpRep, rep, sizeof(tmpRep));
ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/
ZSTD_resetSeqStore(seqStore);
/* invalidate first scan from history */
ms->window.base -= srcSize;
ms->window.dictLimit += (U32)srcSize;
ms->window.lowLimit = ms->window.dictLimit;
ms->nextToUpdate = ms->window.dictLimit;
ms->nextToUpdate3 = ms->window.dictLimit;
/* re-inforce weight of collected statistics */
ZSTD_upscaleStats(&ms->opt);
&& (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */
&& (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */
&& (srcSize > ZSTD_PREDEF_THRESHOLD)
) {
ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize);
}
#endif
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict);
}
@@ -1130,3 +1211,7 @@ size_t ZSTD_compressBlock_btultra_extDict(
{
return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict);
}
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */

8
C/zstd/zstd_opt.h
View File

@@ -26,6 +26,10 @@ size_t ZSTD_compressBlock_btopt(
size_t ZSTD_compressBlock_btultra(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btultra2(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
size_t ZSTD_compressBlock_btopt_dictMatchState(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
@@ -41,6 +45,10 @@ size_t ZSTD_compressBlock_btultra_extDict(
ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
void const* src, size_t srcSize);
/* note : no btultra2 variant for extDict nor dictMatchState,
* because btultra2 is not meant to work with dictionaries
* and is only specific for the first block (no prefix) */
#if defined (__cplusplus)
}
#endif

10
C/zstd/zstd_v04.c
View File

@@ -240,17 +240,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr)
/* *************************************
* Types
***************************************/
#define ZSTD_WINDOWLOG_MAX 26
#define ZSTD_WINDOWLOG_MIN 18
#define ZSTD_WINDOWLOG_ABSOLUTEMIN 11
#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1)
#define ZSTD_CONTENTLOG_MIN 4
#define ZSTD_HASHLOG_MAX 28
#define ZSTD_HASHLOG_MIN 4
#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1)
#define ZSTD_SEARCHLOG_MIN 1
#define ZSTD_SEARCHLENGTH_MAX 7
#define ZSTD_SEARCHLENGTH_MIN 4
/** from faster to stronger */
typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy;

30
C/zstd/zstd_v05.c
View File

@@ -836,7 +836,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits)
bitD->bitsConsumed += nbBits;
}
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits)
{
size_t value = BITv05_lookBits(bitD, nbBits);
BITv05_skipBits(bitD, nbBits);
@@ -845,7 +845,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits)
/*!BITv05_readBitsFast :
* unsafe version; only works only if nbBits >= 1 */
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits)
MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits)
{
size_t value = BITv05_lookBitsFast(bitD, nbBits);
BITv05_skipBits(bitD, nbBits);
@@ -1162,7 +1162,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr)
/* **************************************************************
* Complex types
****************************************************************/
typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)];
/* **************************************************************
@@ -2191,7 +2191,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog,
}
}
size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize)
{
BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1];
sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1];
@@ -2205,7 +2205,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize)
void* dtPtr = DTable;
HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1;
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */
HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */
if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge);
//memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */
@@ -2332,7 +2332,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B
size_t HUFv05_decompress1X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
const unsigned* DTable)
{
const BYTE* const istart = (const BYTE*) cSrc;
BYTE* const ostart = (BYTE*) dst;
@@ -2375,7 +2375,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t
size_t HUFv05_decompress4X4_usingDTable(
void* dst, size_t dstSize,
const void* cSrc, size_t cSrcSize,
const U32* DTable)
const unsigned* DTable)
{
if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */
@@ -2999,7 +2999,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
const BYTE* ip = istart;
const BYTE* const iend = istart + srcSize;
U32 LLtype, Offtype, MLtype;
U32 LLlog, Offlog, MLlog;
unsigned LLlog, Offlog, MLlog;
size_t dumpsLength;
/* check */
@@ -3057,7 +3057,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
{ U32 max = MaxLL;
{ unsigned max = MaxLL;
headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip);
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
if (LLlog > LLFSEv05Log) return ERROR(corruption_detected);
@@ -3081,7 +3081,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
{ U32 max = MaxOff;
{ unsigned max = MaxOff;
headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip);
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
if (Offlog > OffFSEv05Log) return ERROR(corruption_detected);
@@ -3105,7 +3105,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t
break;
case FSEv05_ENCODING_DYNAMIC :
default : /* impossible */
{ U32 max = MaxML;
{ unsigned max = MaxML;
headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip);
if (FSEv05_isError(headerSize)) return ERROR(GENERIC);
if (MLlog > MLFSEv05Log) return ERROR(corruption_detected);
@@ -3305,9 +3305,9 @@ static size_t ZSTDv05_decompressSequences(
const BYTE* const litEnd = litPtr + dctx->litSize;
int nbSeq=0;
const BYTE* dumps = NULL;
U32* DTableLL = dctx->LLTable;
U32* DTableML = dctx->MLTable;
U32* DTableOffb = dctx->OffTable;
unsigned* DTableLL = dctx->LLTable;
unsigned* DTableML = dctx->MLTable;
unsigned* DTableOffb = dctx->OffTable;
const BYTE* const base = (const BYTE*) (dctx->base);
const BYTE* const vBase = (const BYTE*) (dctx->vBase);
const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd);
@@ -3633,7 +3633,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d
{
size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize;
short offcodeNCount[MaxOff+1];
U32 offcodeMaxValue=MaxOff, offcodeLog;
unsigned offcodeMaxValue=MaxOff, offcodeLog;
short matchlengthNCount[MaxML+1];
unsigned matchlengthMaxValue = MaxML, matchlengthLog;
short litlengthNCount[MaxLL+1];

290
C/zstd/zstdmt_compress.c
View File

@@ -9,21 +9,19 @@
*/
/* ====== Tuning parameters ====== */
#define ZSTDMT_NBWORKERS_MAX 200
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */
#define ZSTDMT_OVERLAPLOG_DEFAULT 6
/* ====== Compiler specifics ====== */
#if defined(_MSC_VER)
# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
#endif
/* ====== Constants ====== */
#define ZSTDMT_OVERLAPLOG_DEFAULT 0
/* ====== Dependencies ====== */
#include <string.h> /* memcpy, memset */
#include <limits.h> /* INT_MAX */
#include <limits.h> /* INT_MAX, UINT_MAX */
#include "pool.h" /* threadpool */
#include "threading.h" /* mutex */
#include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */
@@ -58,8 +56,8 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void)
if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK);
{ struct tms junk; clock_t newTicks = (clock_t) times(&junk);
return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); }
}
return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond);
} }
#define MUTEX_WAIT_TIME_DLEVEL 6
#define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \
@@ -342,8 +340,8 @@ static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers)
typedef struct {
ZSTD_pthread_mutex_t poolMutex;
unsigned totalCCtx;
unsigned availCCtx;
int totalCCtx;
int availCCtx;
ZSTD_customMem cMem;
ZSTD_CCtx* cctx[1]; /* variable size */
} ZSTDMT_CCtxPool;
@@ -351,16 +349,16 @@ typedef struct {
/* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */
static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool)
{
unsigned u;
for (u=0; u<pool->totalCCtx; u++)
ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */
int cid;
for (cid=0; cid<pool->totalCCtx; cid++)
ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */
ZSTD_pthread_mutex_destroy(&pool->poolMutex);
ZSTD_free(pool, pool->cMem);
}
/* ZSTDMT_createCCtxPool() :
* implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers,
ZSTD_customMem cMem)
{
ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc(
@@ -381,7 +379,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers,
}
static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool,
unsigned nbWorkers)
int nbWorkers)
{
if (srcPool==NULL) return NULL;
if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */
@@ -469,9 +467,9 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool*
DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10);
ZSTD_ldm_adjustParameters(&params.ldmParams, &params.cParams);
assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog);
assert(params.ldmParams.hashEveryLog < 32);
assert(params.ldmParams.hashRateLog < 32);
serialState->ldmState.hashPower =
ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength);
ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength);
} else {
memset(&params.ldmParams, 0, sizeof(params.ldmParams));
}
@@ -674,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription)
if (ZSTD_isError(initError)) JOB_ERROR(initError);
} else { /* srcStart points at reloaded section */
U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size;
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob);
{ size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob);
if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError);
}
{ size_t const initError = ZSTD_compressBegin_advanced_internal(cctx,
@@ -777,6 +775,14 @@ typedef struct {
static const roundBuff_t kNullRoundBuff = {NULL, 0, 0};
#define RSYNC_LENGTH 32
typedef struct {
U64 hash;
U64 hitMask;
U64 primePower;
} rsyncState_t;
struct ZSTDMT_CCtx_s {
POOL_ctx* factory;
ZSTDMT_jobDescription* jobs;
@@ -790,6 +796,7 @@ struct ZSTDMT_CCtx_s {
inBuff_t inBuff;
roundBuff_t roundBuff;
serialState_t serial;
rsyncState_t rsync;
unsigned singleBlockingThread;
unsigned jobIDMask;
unsigned doneJobID;
@@ -859,7 +866,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker
{
if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX;
params->nbWorkers = nbWorkers;
params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT;
params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT;
params->jobSize = 0;
return nbWorkers;
}
@@ -969,52 +976,59 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx)
}
/* Internal only */
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
ZSTDMT_parameter parameter, unsigned value) {
size_t
ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params,
ZSTDMT_parameter parameter,
int value)
{
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter");
switch(parameter)
{
case ZSTDMT_p_jobSize :
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value);
if ( (value > 0) /* value==0 => automatic job size */
& (value < ZSTDMT_JOBSIZE_MIN) )
DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value);
if ( value != 0 /* default */
&& value < ZSTDMT_JOBSIZE_MIN)
value = ZSTDMT_JOBSIZE_MIN;
if (value > ZSTDMT_JOBSIZE_MAX)
value = ZSTDMT_JOBSIZE_MAX;
assert(value >= 0);
if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX;
params->jobSize = value;
return value;
case ZSTDMT_p_overlapSectionLog :
if (value > 9) value = 9;
DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value);
params->overlapSizeLog = (value >= 9) ? 9 : value;
case ZSTDMT_p_overlapLog :
DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value);
if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN;
if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX;
params->overlapLog = value;
return value;
case ZSTDMT_p_rsyncable :
value = (value != 0);
params->rsyncable = value;
return value;
default :
return ERROR(parameter_unsupported);
}
}
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value)
size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value)
{
DEBUGLOG(4, "ZSTDMT_setMTCtxParameter");
switch(parameter)
{
case ZSTDMT_p_jobSize :
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
case ZSTDMT_p_overlapSectionLog :
return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value);
default :
return ERROR(parameter_unsupported);
}
}
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value)
size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value)
{
switch (parameter) {
case ZSTDMT_p_jobSize:
*value = mtctx->params.jobSize;
assert(mtctx->params.jobSize <= INT_MAX);
*value = (int)(mtctx->params.jobSize);
break;
case ZSTDMT_p_overlapSectionLog:
*value = mtctx->params.overlapSizeLog;
case ZSTDMT_p_overlapLog:
*value = mtctx->params.overlapLog;
break;
case ZSTDMT_p_rsyncable:
*value = mtctx->params.rsyncable;
break;
default:
return ERROR(parameter_unsupported);
@@ -1140,22 +1154,66 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx)
/* ===== Multi-threaded compression ===== */
/* ------------------------------------------ */
static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params)
{
if (params.ldmParams.enableLdm)
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead. */
return MAX(21, params.cParams.chainLog + 4);
return MAX(20, params.cParams.windowLog + 2);
}
static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params)
static int ZSTDMT_overlapLog_default(ZSTD_strategy strat)
{
unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog;
if (params.ldmParams.enableLdm)
return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog);
return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog);
switch(strat)
{
case ZSTD_btultra2:
return 9;
case ZSTD_btultra:
case ZSTD_btopt:
return 8;
case ZSTD_btlazy2:
case ZSTD_lazy2:
return 7;
case ZSTD_lazy:
case ZSTD_greedy:
case ZSTD_dfast:
case ZSTD_fast:
default:;
}
return 6;
}
static unsigned ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) {
static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat)
{
assert(0 <= ovlog && ovlog <= 9);
if (ovlog == 0) return ZSTDMT_overlapLog_default(strat);
return ovlog;
}
static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params)
{
int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy);
int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog);
assert(0 <= overlapRLog && overlapRLog <= 8);
if (params.ldmParams.enableLdm) {
/* In Long Range Mode, the windowLog is typically oversized.
* In which case, it's preferable to determine the jobSize
* based on chainLog instead.
* Then, ovLog becomes a fraction of the jobSize, rather than windowSize */
ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2)
- overlapRLog;
}
assert(0 <= ovLog && ovLog <= 30);
DEBUGLOG(4, "overlapLog : %i", params.overlapLog);
DEBUGLOG(4, "overlap size : %i", 1 << ovLog);
return (ovLog==0) ? 0 : (size_t)1 << ovLog;
}
static unsigned
ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers)
{
assert(nbWorkers>0);
{ size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params);
size_t const jobMaxSize = jobSizeTarget << 2;
@@ -1178,7 +1236,7 @@ static size_t ZSTDMT_compress_advanced_internal(
ZSTD_CCtx_params params)
{
ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params);
size_t const overlapSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
size_t const overlapSize = ZSTDMT_computeOverlapSize(params);
unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers);
size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs;
size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */
@@ -1293,12 +1351,13 @@ size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params,
unsigned overlapLog)
int overlapLog)
{
ZSTD_CCtx_params cctxParams = mtctx->params;
cctxParams.cParams = params.cParams;
cctxParams.fParams = params.fParams;
cctxParams.overlapSizeLog = overlapLog;
assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX);
cctxParams.overlapLog = overlapLog;
return ZSTDMT_compress_advanced_internal(mtctx,
dst, dstCapacity,
src, srcSize,
@@ -1311,8 +1370,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
const void* src, size_t srcSize,
int compressionLevel)
{
U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT;
ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0);
int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy);
params.fParams.contentSizeFlag = 1;
return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog);
}
@@ -1339,8 +1398,8 @@ size_t ZSTDMT_initCStream_internal(
if (params.nbWorkers != mtctx->params.nbWorkers)
CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) );
if (params.jobSize > 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN;
if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX;
mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */
if (mtctx->singleBlockingThread) {
@@ -1375,14 +1434,24 @@ size_t ZSTDMT_initCStream_internal(
mtctx->cdict = cdict;
}
mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params);
DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params);
DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10));
mtctx->targetSectionSize = params.jobSize;
if (mtctx->targetSectionSize == 0) {
mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params);
}
if (params.rsyncable) {
/* Aim for the targetsectionSize as the average job size. */
U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20);
U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20;
assert(jobSizeMB >= 1);
DEBUGLOG(4, "rsyncLog = %u", rsyncBits);
mtctx->rsync.hash = 0;
mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1;
mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH);
}
if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), params.jobSize);
DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize);
DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10));
ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize));
{
@@ -1818,6 +1887,89 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx)
return 1;
}
typedef struct {
size_t toLoad; /* The number of bytes to load from the input. */
int flush; /* Boolean declaring if we must flush because we found a synchronization point. */
} syncPoint_t;
/**
* Searches through the input for a synchronization point. If one is found, we
* will instruct the caller to flush, and return the number of bytes to load.
* Otherwise, we will load as many bytes as possible and instruct the caller
* to continue as normal.
*/
static syncPoint_t
findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input)
{
BYTE const* const istart = (BYTE const*)input.src + input.pos;
U64 const primePower = mtctx->rsync.primePower;
U64 const hitMask = mtctx->rsync.hitMask;
syncPoint_t syncPoint;
U64 hash;
BYTE const* prev;
size_t pos;
syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
syncPoint.flush = 0;
if (!mtctx->params.rsyncable)
/* Rsync is disabled. */
return syncPoint;
if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH)
/* Not enough to compute the hash.
* We will miss any synchronization points in this RSYNC_LENGTH byte
* window. However, since it depends only in the internal buffers, if the
* state is already synchronized, we will remain synchronized.
* Additionally, the probability that we miss a synchronization point is
* low: RSYNC_LENGTH / targetSectionSize.
*/
return syncPoint;
/* Initialize the loop variables. */
if (mtctx->inBuff.filled >= RSYNC_LENGTH) {
/* We have enough bytes buffered to initialize the hash.
* Start scanning at the beginning of the input.
*/
pos = 0;
prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH;
hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH);
} else {
/* We don't have enough bytes buffered to initialize the hash, but
* we know we have at least RSYNC_LENGTH bytes total.
* Start scanning after the first RSYNC_LENGTH bytes less the bytes
* already buffered.
*/
pos = RSYNC_LENGTH - mtctx->inBuff.filled;
prev = (BYTE const*)mtctx->inBuff.buffer.start - pos;
hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled);
hash = ZSTD_rollingHash_append(hash, istart, pos);
}
/* Starting with the hash of the previous RSYNC_LENGTH bytes, roll
* through the input. If we hit a synchronization point, then cut the
* job off, and tell the compressor to flush the job. Otherwise, load
* all the bytes and continue as normal.
* If we go too long without a synchronization point (targetSectionSize)
* then a block will be emitted anyways, but this is okay, since if we
* are already synchronized we will remain synchronized.
*/
for (; pos < syncPoint.toLoad; ++pos) {
BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH];
/* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */
hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower);
if ((hash & hitMask) == hitMask) {
syncPoint.toLoad = pos + 1;
syncPoint.flush = 1;
break;
}
}
return syncPoint;
}
size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx)
{
size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled;
if (hintInSize==0) hintInSize = mtctx->targetSectionSize;
return hintInSize;
}
/** ZSTDMT_compressStream_generic() :
* internal use only - exposed to be invoked from zstd_compress.c
@@ -1844,7 +1996,8 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
}
/* single-pass shortcut (note : synchronous-mode) */
if ( (mtctx->nextJobID == 0) /* just started */
if ( (!mtctx->params.rsyncable) /* rsyncable mode is disabled */
&& (mtctx->nextJobID == 0) /* just started */
&& (mtctx->inBuff.filled == 0) /* nothing buffered */
&& (!mtctx->jobReady) /* no job already created */
&& (endOp == ZSTD_e_end) /* end order */
@@ -1876,14 +2029,17 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx,
DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start);
}
if (mtctx->inBuff.buffer.start != NULL) {
size_t const toLoad = MIN(input->size - input->pos, mtctx->targetSectionSize - mtctx->inBuff.filled);
syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input);
if (syncPoint.flush && endOp == ZSTD_e_continue) {
endOp = ZSTD_e_flush;
}
assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize);
DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u",
(U32)toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad);
input->pos += toLoad;
mtctx->inBuff.filled += toLoad;
forwardInputProgress = toLoad>0;
(U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize);
memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad);
input->pos += syncPoint.toLoad;
mtctx->inBuff.filled += syncPoint.toLoad;
forwardInputProgress = syncPoint.toLoad>0;
}
if ((input->pos < input->size) && (endOp == ZSTD_e_end))
endOp = ZSTD_e_flush; /* can't end now : not all input consumed */

26
C/zstd/zstdmt_compress.h
View File

@@ -28,6 +28,16 @@
#include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */
/* === Constants === */
#ifndef ZSTDMT_NBWORKERS_MAX
# define ZSTDMT_NBWORKERS_MAX 200
#endif
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1 MB)
#endif
#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB))
/* === Memory management === */
typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx;
ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers);
@@ -52,6 +62,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx,
ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel);
ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */
ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx);
ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input);
ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */
@@ -60,16 +71,12 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output);
/* === Advanced functions and parameters === */
#ifndef ZSTDMT_JOBSIZE_MIN
# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */
#endif
ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx,
void* dst, size_t dstCapacity,
const void* src, size_t srcSize,
const ZSTD_CDict* cdict,
ZSTD_parameters params,
unsigned overlapLog);
int overlapLog);
ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx,
const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */
@@ -85,7 +92,8 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx,
* List of parameters that can be set using ZSTDMT_setMTCtxParameter() */
typedef enum {
ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */
ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */
ZSTDMT_p_rsyncable /* Enables rsyncable mode. */
} ZSTDMT_parameter;
/* ZSTDMT_setMTCtxParameter() :
@@ -93,12 +101,12 @@ typedef enum {
* The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__
* Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions.
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value);
ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value);
/* ZSTDMT_getMTCtxParameter() :
* Query the ZSTDMT_CCtx for a parameter value.
* @return : 0, or an error code (which can be tested using ZSTD_isError()) */
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value);
ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value);
/*! ZSTDMT_compressStream_generic() :
@@ -129,7 +137,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx);
/*! ZSTDMT_CCtxParam_setMTCtxParameter()
* like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value);
size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value);
/*! ZSTDMT_CCtxParam_setNbWorkers()
* Set nbWorkers, and clamp it.

2
C/zstdmt/brotli-mt_compress.c
View File

@@ -267,7 +267,7 @@ static void *pt_compress(void *arg)
uint8_t *obuf = (uint8_t*)wl->out.buf + 16;
wl->out.size -= 16;
rv = BrotliEncoderCompress(ctx->level,
BROTLI_LARGE_MAX_WINDOW_BITS,
BROTLI_MAX_WINDOW_BITS,
BROTLI_MODE_GENERIC, in.size,
ibuf, &wl->out.size, obuf);

2
C/zstdmt/lz4-mt_compress.c
View File

@@ -291,11 +291,13 @@ static void *pt_compress(void *arg)
}
/* write skippable frame */
if (ctx->threads > 1) {
MEM_writeLE32((unsigned char *)wl->out.buf + 0,
LZ4FMT_MAGIC_SKIPPABLE);
MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4);
MEM_writeLE32((unsigned char *)wl->out.buf + 8, (U32) result);
wl->out.size = result + 12;
}
/* write result */
pthread_mutex_lock(&ctx->write_mutex);

2
CPP/7zip/Aes.mak
View File

@@ -1,7 +1,7 @@
C_OBJS = $(C_OBJS) \
$O\Aes.obj
!IF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" && "$(CPU)" != "ARM" && "$(CPU)" != "ARM64"
!IF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64"
ASM_OBJS = $(ASM_OBJS) \
$O\AesOpt.obj
!ENDIF

2
CPP/7zip/Archive/7z/7zDecode.cpp
View File

@@ -355,7 +355,9 @@ HRESULT CDecoder::Decode(
unsigned i;
#if !defined(_7ZIP_ST)
bool mt_wasUsed = false;
#endif
for (i = 0; i < folderInfo.Coders.Size(); i++)
{

22
CPP/7zip/Archive/7z/7zHandler.cpp
View File

@@ -236,6 +236,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)
prop = v;
break;
}
case kpidReadOnly:
{
if (!_db.CanUpdate())
prop = true;
break;
}
}
prop.Detach(value);
return S_OK;
@@ -507,7 +514,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
else if (id == k_LZ4)
{
name = "LZ4";
if (propsSize == 5)
if (propsSize == 3 || propsSize == 5)
{
char *dest = s;
*dest++ = 'v';
@@ -525,7 +532,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
else if (id == k_LZ5)
{
name = "LZ5";
if (propsSize == 5)
if (propsSize == 3 || propsSize == 5)
{
char *dest = s;
*dest++ = 'v';
@@ -543,9 +550,10 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
else if (id == k_ZSTD)
{
name = "ZSTD";
if (propsSize == 5)
if (propsSize == 3 || propsSize == 5)
{
char *dest = s;
UInt32 l = props[2];
*dest++ = 'v';
ConvertUInt32ToString(props[0], dest);
dest += MyStringLen(dest);
@@ -553,8 +561,14 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const
ConvertUInt32ToString(props[1], dest);
dest += MyStringLen(dest);
*dest++ = ',';
if (l <= 22) {
*dest++ = 'l';
ConvertUInt32ToString(props[2], dest);
ConvertUInt32ToString(l, dest);
} else {
*dest++ = 'f';
*dest++ = 'l';
ConvertUInt32ToString(l - 32, dest);
}
dest += MyStringLen(dest);
}
}

Some files were not shown because too many files have changed in this diff Show More