diff --git a/C/7zArcIn.c b/C/7zArcIn.c index 70d7175e..68cc12ff 100644 --- a/C/7zArcIn.c +++ b/C/7zArcIn.c @@ -1,5 +1,5 @@ /* 7zArcIn.c -- 7z Input functions -2017-04-03 : Igor Pavlov : Public domain */ +2018-12-31 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -19,7 +19,7 @@ { MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } #define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ - { if ((size) == 0) p = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } + { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } #define k7zMajorVersion 0 @@ -666,7 +666,7 @@ static SRes ReadUnpackInfo(CSzAr *p, MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); startBufPtr = sd.Data; @@ -1744,7 +1744,7 @@ size_t SzArEx_GetFullNameLen(const CSzArEx *p, size_t fileIndex) UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) { - Bool needSlash; + BoolInt needSlash; if (!p->FileNameOffsets) { *(--dest) = 0; diff --git a/C/7zDec.c b/C/7zDec.c index 9c986950..2a7b0903 100644 --- a/C/7zDec.c +++ b/C/7zDec.c @@ -1,5 +1,5 @@ /* 7zDec.c -- Decoding from 7z folder -2017-04-03 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -44,7 +44,7 @@ typedef struct const Byte *end; const Byte *begin; UInt64 processed; - Bool extra; + BoolInt extra; SRes res; const ILookInStream *inStream; } CByteInToLook; @@ -156,7 +156,7 @@ static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, I { SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -218,7 +218,7 @@ static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, { SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; ELzmaStatus status; - res = Lzma2Dec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK) @@ -269,7 +269,7 @@ static SRes SzDecodeCopy(UInt64 inSize, ILookInStream *inStream, Byte *outBuffer return SZ_OK; } -static Bool IS_MAIN_METHOD(UInt32 m) +static BoolInt IS_MAIN_METHOD(UInt32 m) { switch (m) { @@ -286,7 +286,7 @@ static Bool IS_MAIN_METHOD(UInt32 m) return False; } -static Bool IS_SUPPORTED_CODER(const CSzCoderInfo *c) +static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) { return c->NumStreams == 1 diff --git a/C/7zTypes.h b/C/7zTypes.h index 4977cdaa..593f5aa2 100644 --- a/C/7zTypes.h +++ b/C/7zTypes.h @@ -1,5 +1,5 @@ /* 7zTypes.h -- Basic types -2017-07-17 : Igor Pavlov : Public domain */ +2018-08-04 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H @@ -103,7 +103,8 @@ typedef UInt32 SizeT; typedef size_t SizeT; #endif -typedef int Bool; +typedef int BoolInt; +/* typedef BoolInt Bool; */ #define True 1 #define False 0 diff --git a/C/7zVersion.h b/C/7zVersion.h index d4db5505..37b48d9d 100644 --- a/C/7zVersion.h +++ b/C/7zVersion.h @@ -1,7 +1,7 @@ -#define MY_VER_MAJOR 18 -#define MY_VER_MINOR 05 +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "18.05 ZS v1.3.7 R3" +#define MY_VERSION_NUMBERS "19.00 ZS v1.3.8 R2" #define MY_VERSION MY_VERSION_NUMBERS #ifdef MY_CPU_NAME @@ -10,12 +10,12 @@ #define MY_VERSION_CPU MY_VERSION #endif -#define MY_DATE "2018-10-28" +#define MY_DATE "2019-03-05" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov, Tino Reichardt" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov, 2016-2018 Tino Reichardt" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2019 Igor Pavlov, 2016-2019 Tino Reichardt" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR diff --git a/C/7zVersionTr.h b/C/7zVersionTr.h index a7a83afa..6ace93fb 100644 --- a/C/7zVersionTr.h +++ b/C/7zVersionTr.h @@ -1,11 +1,14 @@ -#define MY_VER_MAJOR 18 -#define MY_VER_MINOR 05 -#define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "1.3.7 R3" -#define MY_VERSION MY_VERSION_NUMBERS -#define MY_DATE "2018-10-28" -#undef MY_COPYRIGHT -#undef MY_VERSION_COPYRIGHT_DATE -#define MY_AUTHOR_NAME "Tino Reichardt" -#define MY_COPYRIGHT "Copyright (c) 2016 - 2018 Tino Reichardt" -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " : " MY_COPYRIGHT " : " MY_DATE + +#include "7zVersion.h" + +#undef MY_AUTHOR_NAME +#define MY_AUTHOR_NAME "Tino Reichardt" + +#undef MY_COPYRIGHT +#define MY_COPYRIGHT "Copyright (c) 2016 - 2019 Tino Reichardt" + +#undef MY_COPYRIGHT_DATE +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE + +#undef MY_VERSION_COPYRIGHT_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE diff --git a/C/Bcj2Enc.c b/C/Bcj2Enc.c index b0bc7593..7a02ecde 100644 --- a/C/Bcj2Enc.c +++ b/C/Bcj2Enc.c @@ -1,5 +1,5 @@ /* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2017-04-28 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -52,7 +52,7 @@ void Bcj2Enc_Init(CBcj2Enc *p) p->probs[i] = kBitModelTotal >> 1; } -static Bool MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) { if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) { @@ -165,7 +165,7 @@ static void Bcj2Enc_Encode_2(CBcj2Enc *p) { Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - Bool needConvert; + BoolInt needConvert; p->bufs[BCJ2_STREAM_MAIN] = dest + 1; p->ip += (UInt32)num + 1; @@ -253,7 +253,7 @@ void Bcj2Enc_Encode(CBcj2Enc *p) { const Byte *src = p->src; const Byte *srcLim = p->srcLim; - unsigned finishMode = p->finishMode; + EBcj2Enc_FinishMode finishMode = p->finishMode; p->src = p->temp; p->srcLim = p->temp + p->tempPos; diff --git a/C/BwtSort.c b/C/BwtSort.c index 87a612e4..a87e4d17 100644 --- a/C/BwtSort.c +++ b/C/BwtSort.c @@ -1,5 +1,5 @@ /* BwtSort.c -- BWT block sorting -2017-04-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -448,7 +448,7 @@ UInt32 BlockSort(UInt32 *Indices, const Byte *data, UInt32 blockSize) groupSize = ((Indices[i] & ~0xC0000000) >> kNumBitsMax); { - Bool finishedGroup = ((Indices[i] & 0x80000000) == 0); + BoolInt finishedGroup = ((Indices[i] & 0x80000000) == 0); if ((Indices[i] & 0x40000000) != 0) { groupSize += ((Indices[(size_t)i + 1] >> kNumBitsMax) << kNumExtra0Bits); diff --git a/C/CpuArch.c b/C/CpuArch.c index f835c2b7..ff1890e7 100644 --- a/C/CpuArch.c +++ b/C/CpuArch.c @@ -1,5 +1,5 @@ /* CpuArch.c -- CPU specific code -2016-02-25: Igor Pavlov : Public domain */ +2018-02-18: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -115,7 +115,7 @@ void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d) #endif } -Bool x86cpuid_CheckAndRead(Cx86cpuid *p) +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) { CHECK_CPUID_IS_SUPPORTED MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]); @@ -144,7 +144,7 @@ int x86cpuid_GetFirm(const Cx86cpuid *p) return -1; } -Bool CPU_Is_InOrder() +BoolInt CPU_Is_InOrder() { Cx86cpuid p; int firm; @@ -175,7 +175,7 @@ Bool CPU_Is_InOrder() #if !defined(MY_CPU_AMD64) && defined(_WIN32) #include -static Bool CPU_Sys_Is_SSE_Supported() +static BoolInt CPU_Sys_Is_SSE_Supported() { OSVERSIONINFO vi; vi.dwOSVersionInfoSize = sizeof(vi); @@ -188,7 +188,7 @@ static Bool CPU_Sys_Is_SSE_Supported() #define CHECK_SYS_SSE_SUPPORT #endif -Bool CPU_Is_Aes_Supported() +BoolInt CPU_Is_Aes_Supported() { Cx86cpuid p; CHECK_SYS_SSE_SUPPORT @@ -197,4 +197,22 @@ Bool CPU_Is_Aes_Supported() return (p.c >> 25) & 1; } +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } +} + #endif diff --git a/C/CpuArch.h b/C/CpuArch.h index 7fb27282..5f74c1c0 100644 --- a/C/CpuArch.h +++ b/C/CpuArch.h @@ -1,5 +1,5 @@ /* CpuArch.h -- CPU specific code -2017-09-04 : Igor Pavlov : Public domain */ +2018-02-18 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H @@ -318,15 +318,16 @@ enum void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); -Bool x86cpuid_CheckAndRead(Cx86cpuid *p); +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); int x86cpuid_GetFirm(const Cx86cpuid *p); #define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) #define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) #define x86cpuid_GetStepping(ver) (ver & 0xF) -Bool CPU_Is_InOrder(); -Bool CPU_Is_Aes_Supported(); +BoolInt CPU_Is_InOrder(); +BoolInt CPU_Is_Aes_Supported(); +BoolInt CPU_IsSupported_PageGB(); #endif diff --git a/C/DllSecur.c b/C/DllSecur.c index 8745421a..19a22a9f 100644 --- a/C/DllSecur.c +++ b/C/DllSecur.c @@ -1,5 +1,5 @@ /* DllSecur.c -- DLL loading security -2016-10-04 : Igor Pavlov : Public domain */ +2018-02-21 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -28,10 +28,31 @@ static const char * const g_Dlls = "CRYPTBASE\0" "OLEACC\0" "CLBCATQ\0" + "VERSION\0" ; #endif +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + GetVersionEx(&vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + void LoadSecurityDlls() { #ifndef UNDER_CE @@ -70,7 +91,7 @@ void LoadSecurityDlls() for (;;) { char c = *dll++; - buf[pos + k] = c; + buf[pos + k] = (Byte)c; k++; if (c == 0) break; diff --git a/C/DllSecur.h b/C/DllSecur.h index 023c5096..4c113568 100644 --- a/C/DllSecur.h +++ b/C/DllSecur.h @@ -1,5 +1,5 @@ /* DllSecur.h -- DLL loading for security -2016-06-08 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __DLL_SECUR_H #define __DLL_SECUR_H @@ -10,6 +10,7 @@ EXTERN_C_BEGIN #ifdef _WIN32 +void My_SetDefaultDllDirectories(); void LoadSecurityDlls(); #endif diff --git a/C/LzFind.c b/C/LzFind.c index 6ea82a9b..4eefc17d 100644 --- a/C/LzFind.c +++ b/C/LzFind.c @@ -1,5 +1,5 @@ /* LzFind.c -- Match finder for LZ algorithms -2017-06-10 : Igor Pavlov : Public domain */ +2018-07-08 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -138,7 +138,7 @@ static void MatchFinder_SetDefaultSettings(CMatchFinder *p) void MatchFinder_Construct(CMatchFinder *p) { - UInt32 i; + unsigned i; p->bufferBase = NULL; p->directInput = 0; p->hash = NULL; @@ -147,7 +147,7 @@ void MatchFinder_Construct(CMatchFinder *p) for (i = 0; i < 256; i++) { - UInt32 r = i; + UInt32 r = (UInt32)i; unsigned j; for (j = 0; j < 8; j++) r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); @@ -368,6 +368,8 @@ static void MatchFinder_Normalize(CMatchFinder *p) MatchFinder_ReduceOffsets(p, subValue); } + +MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { if (p->pos == kMaxValForNormalize) @@ -379,10 +381,16 @@ static void MatchFinder_CheckLimits(CMatchFinder *p) MatchFinder_SetLimits(p); } -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + UInt32 *distances, unsigned maxLen) { + /* son[_cyclicBufferPos] = curMatch; for (;;) { @@ -400,7 +408,8 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = len; *distances++ = delta - 1; if (len == lenLimit) return distances; @@ -408,15 +417,58 @@ static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, } } } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + do + { + UInt32 delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - delta; + if (cur[maxLen] == cur[maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + distances[0] = (UInt32)(lim - cur); + distances[1] = delta - 1; + return distances + 2; + } + } + { + unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + distances[0] = (UInt32)len; + distances[1] = delta - 1; + distances += 2; + } + } + } + } + } + while (--cutValue); + + return distances; } + +MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, UInt32 *distances, UInt32 maxLen) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch; @@ -426,9 +478,10 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt return distances; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = pair[0]; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -437,11 +490,12 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = (UInt32)len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; + *ptr1 = pair0; *ptr0 = pair[1]; return distances; } @@ -468,9 +522,9 @@ UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byt static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch; @@ -480,9 +534,9 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const return; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); if (pb[len] == cur[len]) { while (++len != lenLimit) @@ -520,13 +574,13 @@ static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const p->buffer++; \ if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); -#define MOVE_POS_RET MOVE_POS return offset; +#define MOVE_POS_RET MOVE_POS return (UInt32)offset; static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; #define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0) @@ -535,22 +589,22 @@ static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue #define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; + offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ + distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; #define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; #define UPDATE_maxLen { \ ptrdiff_t diff = (ptrdiff_t)0 - d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ - maxLen = (UInt32)(c - cur); } + maxLen = (unsigned)(c - cur); } static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(2) HASH2_CALC; curMatch = p->hash[hv]; @@ -561,7 +615,7 @@ static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; @@ -572,7 +626,8 @@ UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, d2, maxLen, offset, pos; + UInt32 h2, d2, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(3) @@ -594,12 +649,12 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { UPDATE_maxLen - distances[0] = maxLen; + distances[0] = (UInt32)maxLen; distances[1] = d2 - 1; offset = 2; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } } @@ -609,7 +664,8 @@ static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -618,12 +674,12 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; @@ -632,7 +688,8 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; } @@ -648,10 +705,10 @@ static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } } @@ -674,13 +731,13 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; d4 = pos - (hash + kFix4HashSize)[h4]; curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; @@ -741,7 +798,8 @@ static UInt32 Bt5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4) @@ -750,12 +808,11 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; - curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[hv] = pos; @@ -764,7 +821,8 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; } @@ -780,7 +838,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (offset != 0) { UPDATE_maxLen - distances[(size_t)offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch; @@ -791,7 +849,7 @@ static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) if (maxLen < 3) maxLen = 3; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances + offset, maxLen) - (distances)); MOVE_POS_RET } @@ -808,13 +866,13 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; + d2 = pos - hash [h2]; d3 = pos - (hash + kFix3HashSize)[h3]; d4 = pos - (hash + kFix4HashSize)[h4]; curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; + hash [h2] = pos; (hash + kFix3HashSize)[h3] = pos; (hash + kFix4HashSize)[h4] = pos; (hash + kFix5HashSize)[hv] = pos; @@ -877,12 +935,12 @@ static UInt32 Hc5_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances, 2) - (distances)); MOVE_POS_RET } @@ -940,7 +998,7 @@ static void Bt4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH4_CALC; hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[hv] = p->pos; SKIP_FOOTER @@ -959,7 +1017,7 @@ static void Bt5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH5_CALC; hash = p->hash; curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; @@ -979,7 +1037,7 @@ static void Hc4_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH4_CALC; hash = p->hash; curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; @@ -999,7 +1057,7 @@ static void Hc5_MatchFinder_Skip(CMatchFinder *p, UInt32 num) HASH5_CALC; hash = p->hash; curMatch = hash + kFix5HashSize)[hv]; - hash[ h2] = + hash [h2] = (hash + kFix3HashSize)[h3] = (hash + kFix4HashSize)[h4] = (hash + kFix5HashSize)[hv] = p->pos; diff --git a/C/LzFindMt.c b/C/LzFindMt.c index 2563824f..df32146f 100644 --- a/C/LzFindMt.c +++ b/C/LzFindMt.c @@ -1,5 +1,5 @@ /* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2017-06-10 : Igor Pavlov : Public domain */ +2018-12-29 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -232,38 +232,57 @@ static void MatchFinderMt_GetNextBlock_Hash(CMatchFinderMt *p) #define kEmptyHashValue 0 -/* #define MFMT_GM_INLINE */ +#define MFMT_GM_INLINE #ifdef MFMT_GM_INLINE -#define NO_INLINE MY_FAST_CALL +/* + we use size_t for _cyclicBufferPos instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ -static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes) +MY_NO_INLINE +static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + UInt32 *distances, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) { do { - UInt32 *distances = _distances + 1; - UInt32 curMatch = pos - *hash++; + UInt32 *_distances = ++distances; + UInt32 delta = *hash++; - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; UInt32 cutValue = _cutValue; - UInt32 maxLen = _maxLen; - for (;;) + unsigned maxLen = (unsigned)_maxLen; + + /* + if (size > 1) { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + UInt32 delta = *hash; + if (delta < _cyclicBufferSize) { - *ptr0 = *ptr1 = kEmptyHashValue; - break; + UInt32 cyc1 = _cyclicBufferPos + 1; + CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1); + Byte b = *(cur + 1 - delta); + _distances[0] = pair[0]; + _distances[1] = b; } + } + */ + if (cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + } + else + for(;;) + { { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = *pair; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len]) @@ -272,54 +291,66 @@ static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte * break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; - *ptr0 = pair[1]; + UInt32 pair1 = pair[1]; + *ptr1 = pair0; + *ptr0 = pair1; break; } } } - if (pb[len] < cur[len]) { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else - { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; + UInt32 curMatch = pos - delta; + // delta = pos - *pair; + // delta = pos - pair[((UInt32)pb[len] - (UInt32)cur[len]) >> 31]; + if (pb[len] < cur[len]) + { + delta = pos - pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = pos - *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } } } + if (--cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + break; + } } pos++; _cyclicBufferPos++; cur++; { UInt32 num = (UInt32)(distances - _distances); - *_distances = num - 1; - _distances += num; - limit -= num; + _distances[-1] = num; } } - while (limit > 0 && --size != 0); + while (distances < limit && --size != 0); *posRes = pos; - return limit; + return distances; } #endif + + static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) { UInt32 numProcessed = 0; UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); + UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2 distances[1] = p->hashNumAvail; @@ -369,8 +400,10 @@ static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) #else { UInt32 posRes; - curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes); + curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, + distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + distances + limit, + size, &posRes) - distances); p->hashBufPos += posRes - pos; cyclicBufferPos += posRes - pos; p->buffer += posRes - pos; diff --git a/C/LzFindMt.h b/C/LzFindMt.h index 3d86c788..fdd17008 100644 --- a/C/LzFindMt.h +++ b/C/LzFindMt.h @@ -1,5 +1,5 @@ /* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2017-04-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H @@ -19,10 +19,10 @@ EXTERN_C_BEGIN typedef struct _CMtSync { - Bool wasCreated; - Bool needStart; - Bool exit; - Bool stopWriting; + BoolInt wasCreated; + BoolInt needStart; + BoolInt exit; + BoolInt stopWriting; CThread thread; CAutoResetEvent canStart; @@ -30,8 +30,8 @@ typedef struct _CMtSync CAutoResetEvent wasStopped; CSemaphore freeSemaphore; CSemaphore filledSemaphore; - Bool csWasInitialized; - Bool csWasEntered; + BoolInt csWasInitialized; + BoolInt csWasEntered; CCriticalSection cs; UInt32 numProcessedBlocks; } CMtSync; diff --git a/C/Lzma2Dec.c b/C/Lzma2Dec.c index 57e7f346..2e631051 100644 --- a/C/Lzma2Dec.c +++ b/C/Lzma2Dec.c @@ -1,5 +1,5 @@ /* Lzma2Dec.c -- LZMA2 Decoder -2018-02-19 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */ @@ -169,7 +169,7 @@ static void LzmaDec_UpdateWithUncompressed(CLzmaDec *p, const Byte *src, SizeT s p->processedPos += (UInt32)size; } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, @@ -232,7 +232,7 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); + BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); LzmaDec_InitDicAndState(&p->decoder, initDic, False); } @@ -254,8 +254,8 @@ SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, if (p->state == LZMA2_STATE_DATA) { - Bool initDic = (p->control >= 0xE0); - Bool initState = (p->control >= 0xA0); + BoolInt initDic = (p->control >= 0xE0); + BoolInt initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); p->state = LZMA2_STATE_DATA_CONT; } @@ -314,15 +314,15 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, while (p->state != LZMA2_STATE_ERROR) { if (p->state == LZMA2_STATE_FINISHED) - return LZMA_STATUS_FINISHED_WITH_MARK; + return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; if (outSize == 0 && !checkFinishBlock) - return LZMA_STATUS_NOT_FINISHED; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) { if (*srcLen == inSize) - return LZMA_STATUS_NEEDS_MORE_INPUT; + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; (*srcLen)++; p->state = Lzma2Dec_UpdateState(p, *src++); @@ -344,7 +344,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, // checkFinishBlock is true. So we expect that block must be finished, // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here // break; - return LZMA_STATUS_NOT_FINISHED; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; } if (p->state == LZMA2_STATE_DATA) @@ -354,7 +354,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, } if (outSize == 0) - return LZMA_STATUS_NOT_FINISHED; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; { SizeT inCur = inSize - *srcLen; @@ -362,7 +362,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { if (inCur == 0) - return LZMA_STATUS_NEEDS_MORE_INPUT; + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; if (inCur > p->unpackSize) inCur = p->unpackSize; if (inCur > outSize) @@ -381,7 +381,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, if (inCur == 0) { if (p->packSize != 0) - return LZMA_STATUS_NEEDS_MORE_INPUT; + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; } else if (p->state == LZMA2_STATE_DATA) { @@ -418,7 +418,7 @@ ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, } p->state = LZMA2_STATE_ERROR; - return LZMA_STATUS_NOT_SPECIFIED; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; } diff --git a/C/Lzma2DecMt.c b/C/Lzma2DecMt.c index be698cbe..87d5567a 100644 --- a/C/Lzma2DecMt.c +++ b/C/Lzma2DecMt.c @@ -1,5 +1,5 @@ /* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread -2018-03-02 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -94,13 +94,13 @@ typedef struct ISeqOutStream *outStream; ICompressProgress *progress; - Bool finishMode; - Bool outSize_Defined; + BoolInt finishMode; + BoolInt outSize_Defined; UInt64 outSize; UInt64 outProcessed; UInt64 inProcessed; - Bool readWasFinished; + BoolInt readWasFinished; SRes readRes; Byte *inBuf; @@ -113,7 +113,7 @@ typedef struct #ifndef _7ZIP_ST UInt64 outProcessed_Parse; - Bool mtc_WasConstructed; + BoolInt mtc_WasConstructed; CMtDec mtc; CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; #endif @@ -265,7 +265,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa t->outPreSize = 0; // t->blockWasFinished = False; // t->finishedWithMark = False; - t->parseStatus = LZMA_STATUS_NOT_SPECIFIED; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; t->state = MTDEC_PARSE_CONTINUE; t->inCodeSize = 0; @@ -277,7 +277,7 @@ static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCa { ELzma2ParseStatus status; - Bool overflow; + BoolInt overflow; UInt32 unpackRem = 0; int checkFinishBlock = True; @@ -477,7 +477,7 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, { ELzmaStatus status; size_t srcProcessed = srcSize; - Bool blockWasFinished = + BoolInt blockWasFinished = ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); @@ -526,15 +526,15 @@ static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, #define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, - Bool needWriteToStream, + BoolInt needWriteToStream, const Byte *src, size_t srcSize, - Bool *needContinue, Bool *canRecode) + BoolInt *needContinue, BoolInt *canRecode) { CLzma2DecMt *me = (CLzma2DecMt *)pp; const CLzma2DecMtThread *t = &me->coders[coderIndex]; size_t size = t->outCodeSize; const Byte *data = t->outBuf; - Bool needContinue2 = True; + BoolInt needContinue2 = True; PRF_STR_INT_2("Write", coderIndex, srcSize); @@ -633,7 +633,7 @@ static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p #ifndef _7ZIP_ST - , Bool tMode + , BoolInt tMode #endif ) { @@ -674,8 +674,8 @@ static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p SRes res; SizeT outProcessed; - Bool outFinished; - Bool needStop; + BoolInt outFinished; + BoolInt needStop; if (inPos == inLim) { @@ -810,7 +810,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, { CLzma2DecMt *p = (CLzma2DecMt *)pp; #ifndef _7ZIP_ST - Bool tMode; + BoolInt tMode; #endif *inProcessed = 0; @@ -903,7 +903,7 @@ SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, vt.Write = Lzma2DecMt_MtCallback_Write; { - Bool needContinue = False; + BoolInt needContinue = False; SRes res = MtDec_Code(&p->mtc); diff --git a/C/Lzma2Enc.c b/C/Lzma2Enc.c index 5098195c..d5414775 100644 --- a/C/Lzma2Enc.c +++ b/C/Lzma2Enc.c @@ -1,5 +1,5 @@ /* Lzma2Enc.c -- LZMA2 Encoder -2018-04-27 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -115,7 +115,7 @@ SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 k ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); void LzmaEnc_Finish(CLzmaEncHandle pp); @@ -133,7 +133,7 @@ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t packSize = packSizeLimit; UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); - Bool useCopyBlock; + BoolInt useCopyBlock; SRes res; *packSizeRes = 0; @@ -373,7 +373,7 @@ typedef struct size_t outBufSize; /* size of allocated outBufs[i] */ size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; - Bool mtCoder_WasConstructed; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; Byte *outBufs[MTCODER__BLOCKS_MAX]; diff --git a/C/Lzma86Enc.c b/C/Lzma86Enc.c index ee59fb7d..8d35e6dc 100644 --- a/C/Lzma86Enc.c +++ b/C/Lzma86Enc.c @@ -1,5 +1,5 @@ /* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2016-05-16 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -18,7 +18,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, { size_t outSize2 = *destLen; Byte *filteredStream; - Bool useFilter; + BoolInt useFilter; int mainResult = SZ_ERROR_OUTPUT_EOF; CLzmaEncProps props; LzmaEncProps_Init(&props); @@ -56,7 +56,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, { size_t minSize = 0; - Bool bestIsFiltered = False; + BoolInt bestIsFiltered = False; /* passes for SZ_FILTER_AUTO: 0 - BCJ + LZMA @@ -71,7 +71,7 @@ int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; size_t outPropsSize = 5; SRes curRes; - Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); + BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); if (curModeIsFiltered && !bestIsFiltered) break; if (useFilter && i == 0) diff --git a/C/LzmaDec.c b/C/LzmaDec.c index 962b94bb..4d157641 100644 --- a/C/LzmaDec.c +++ b/C/LzmaDec.c @@ -1,13 +1,13 @@ /* LzmaDec.c -- LZMA Decoder -2018-02-28 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include + /* #include "CpuArch.h" */ #include "LzmaDec.h" -#include - #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits) @@ -19,7 +19,7 @@ #define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ @@ -66,7 +66,7 @@ #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \ @@ -539,7 +539,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit curLen = ((rem < len) ? (unsigned)rem : len); pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); - processedPos += curLen; + processedPos += (UInt32)curLen; len -= curLen; if (curLen <= dicBufSize - pos) @@ -547,7 +547,7 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit Byte *dest = dic + dicPos; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; const Byte *lim = dest + curLen; - dicPos += curLen; + dicPos += (SizeT)curLen; do *(dest) = (Byte)*(dest + src); while (++dest != lim); @@ -572,14 +572,14 @@ int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit p->buf = buf; p->range = range; p->code = code; - p->remainLen = len; + p->remainLen = (UInt32)len; p->dicPos = dicPos; p->processedPos = processedPos; p->reps[0] = rep0; p->reps[1] = rep1; p->reps[2] = rep2; p->reps[3] = rep3; - p->state = state; + p->state = (UInt32)state; return SZ_OK; } @@ -601,8 +601,8 @@ static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) p->checkDicSize = p->prop.dicSize; - p->processedPos += len; - p->remainLen -= len; + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; while (len != 0) { len--; @@ -850,7 +850,7 @@ static ELzmaDummy LzmaDec_TryDummy(const CLzmaDec *p, const Byte *buf, SizeT inS } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; @@ -979,10 +979,10 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr p->tempBufSize = rem; if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); if (dummyRes == DUMMY_ERROR) { - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; } @@ -1005,9 +1005,9 @@ SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *sr return SZ_ERROR_FAIL; /* some internal error */ lookAhead -= rem; } - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; src += lookAhead; - inSize -= lookAhead; + inSize -= (SizeT)lookAhead; p->tempBufSize = 0; } } diff --git a/C/LzmaEnc.c b/C/LzmaEnc.c index bebe664d..14086fc4 100644 --- a/C/LzmaEnc.c +++ b/C/LzmaEnc.c @@ -1,5 +1,5 @@ /* LzmaEnc.c -- LZMA Encoder -2018-04-29 : Igor Pavlov : Public domain */ +2019-01-10: Igor Pavlov : Public domain */ #include "Precomp.h" @@ -38,6 +38,8 @@ static unsigned g_STAT_OFFSET = 0; #define kNumBitPriceShiftBits 4 #define kBitPrice (1 << kNumBitPriceShiftBits) +#define REP_LEN_COUNT 64 + void LzmaEncProps_Init(CLzmaEncProps *p) { p->level = 5; @@ -183,8 +185,11 @@ typedef struct } COptimal; -#define kNumOpts (1 << 12) -#define kPackReserve (1 + kNumOpts * 2) +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 @@ -236,10 +241,18 @@ typedef struct typedef struct { unsigned tableSize; - unsigned counters[LZMA_NUM_PB_STATES_MAX]; UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; } CLenPriceEnc; +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ typedef struct { @@ -308,16 +321,18 @@ typedef struct unsigned lc, lp, pb; unsigned lclp; - Bool fastMode; - Bool writeEndMark; - Bool finished; - Bool multiThread; - Bool needInit; + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; UInt64 nowPos64; unsigned matchPriceCount; - unsigned alignPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; unsigned distTableSize; @@ -325,7 +340,7 @@ typedef struct SRes result; #ifndef _7ZIP_ST - Bool mtMode; + BoolInt mtMode; // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; // end of CMatchFinderMt is used in BT and HASH threads @@ -458,6 +473,7 @@ SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) p->lp = props.lp; p->pb = props.pb; p->fastMode = (props.algo == 0); + // p->_maxMode = True; p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); { unsigned numHashBytes = 4; @@ -618,9 +634,9 @@ static void RangeEnc_FlushData(CRangeEnc *p) #ifdef _LZMA_ENC_USE_BRANCH -#define RC_BIT(p, prob, symbol) { \ +#define RC_BIT(p, prob, bit) { \ RC_BIT_PRE(p, prob) \ - if (symbol == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ *(prob) = (CLzmaProb)ttt; \ RC_NORM(p) \ @@ -628,15 +644,15 @@ static void RangeEnc_FlushData(CRangeEnc *p) #else -#define RC_BIT(p, prob, symbol) { \ +#define RC_BIT(p, prob, bit) { \ UInt32 mask; \ RC_BIT_PRE(p, prob) \ - mask = 0 - (UInt32)symbol; \ + mask = 0 - (UInt32)bit; \ range &= mask; \ mask &= newBound; \ range -= mask; \ (p)->low += mask; \ - mask = (UInt32)symbol - 1; \ + mask = (UInt32)bit - 1; \ range += newBound & mask; \ mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ mask += ((1 << kNumMoveBits) - 1); \ @@ -673,42 +689,42 @@ static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) p->range = range; } -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) { UInt32 range = p->range; - symbol |= 0x100; + sym |= 0x100; do { UInt32 ttt, newBound; - // RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - CLzmaProb *prob = probs + (symbol >> 8); - UInt32 bit = (symbol >> 7) & 1; - symbol <<= 1; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); p->range = range; } -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) { UInt32 range = p->range; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { UInt32 ttt, newBound; CLzmaProb *prob; UInt32 bit; matchByte <<= 1; - // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - prob = probs + (offs + (matchByte & offs) + (symbol >> 8)); - bit = (symbol >> 7) & 1; - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); p->range = range; } @@ -739,11 +755,11 @@ static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) } -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (unsigned)(((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ (unsigned)((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] @@ -752,48 +768,48 @@ static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) #define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const CProbPrice *ProbPrices) +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) { UInt32 price = 0; - symbol |= 0x100; + sym |= 0x100; do { - unsigned bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[symbol], bit); + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); } - while (symbol >= 2); + while (sym >= 2); return price; } -static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const CProbPrice *ProbPrices) +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) { UInt32 price = 0; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); } - while (symbol < 0x10000); + while (sym < 0x10000); return price; } -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, UInt32 symbol) +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) { UInt32 range = rc->range; unsigned m = 1; do { UInt32 ttt, newBound; - unsigned bit = symbol & 1; + unsigned bit = sym & 1; // RangeEnc_EncodeBit(rc, probs + m, bit); - symbol >>= 1; + sym >>= 1; RC_BIT(rc, probs + m, bit); m = (m << 1) | bit; } @@ -812,37 +828,37 @@ static void LenEnc_Init(CLenEnc *p) p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned symbol, unsigned posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) { UInt32 range, ttt, newBound; CLzmaProb *probs = p->low; range = rc->range; RC_BIT_PRE(rc, probs); - if (symbol >= kLenNumLowSymbols) + if (sym >= kLenNumLowSymbols) { RC_BIT_1(rc, probs); probs += kLenNumLowSymbols; RC_BIT_PRE(rc, probs); - if (symbol >= kLenNumLowSymbols * 2) + if (sym >= kLenNumLowSymbols * 2) { RC_BIT_1(rc, probs); rc->range = range; - // RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols * 2); - LitEnc_Encode(rc, p->high, symbol - kLenNumLowSymbols * 2); + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); return; } - symbol -= kLenNumLowSymbols; + sym -= kLenNumLowSymbols; } - // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, symbol); + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); { unsigned m; unsigned bit; RC_BIT_0(rc, probs); probs += (posState << (1 + kLenNumLowBits)); - bit = (symbol >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; - bit = (symbol >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; - bit = symbol & 1; RC_BIT(rc, probs + m, bit); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); rc->range = range; } } @@ -863,51 +879,94 @@ static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *price } -MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTable( - CLenPriceEnc *p, unsigned posState, +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, const CLenEnc *enc, const CProbPrice *ProbPrices) { - // int y; for (y = 0; y < 100; y++) { - UInt32 a; - unsigned i, numSymbols; - - UInt32 *prices = p->prices[posState]; + UInt32 b; + { - const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); - SetPrices_3(probs, GET_PRICEa_0(enc->low[0]), prices, ProbPrices); + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (posState = 0; posState < numPosStates; posState++) + { + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); + } + } + + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; a = GET_PRICEa_1(enc->low[0]); - SetPrices_3(probs + kLenNumLowSymbols, a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]), prices + kLenNumLowSymbols, ProbPrices); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); } - numSymbols = p->tableSize; - p->counters[posState] = numSymbols; - for (i = kLenNumLowSymbols * 2; i < numSymbols; i += 1) + */ + + // p->counter = numSymbols; + // p->counter = 64; + { - prices[i] = a + - // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); - LitEnc_GetPrice(enc->high, i - kLenNumLowSymbols * 2, ProbPrices); - /* - unsigned sym = (i - kLenNumLowSymbols * 2) >> 1; - UInt32 price = a + RcTree_GetPrice(enc->high, kLenNumHighBits - 1, sym, ProbPrices); - UInt32 prob = enc->high[(1 << 7) + sym]; - prices[i ] = price + GET_PRICEa_0(prob); - prices[i + 1] = price + GET_PRICEa_1(prob); - */ + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) + { + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); + + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } + } } - // } } -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, unsigned numPosStates, - const CLenEnc *enc, - const CProbPrice *ProbPrices) -{ - unsigned posState; - for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, enc, ProbPrices); -} - - /* #ifdef SHOW_STAT g_STAT_OFFSET += num; @@ -917,7 +976,7 @@ static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, unsigned numPosStates, #define MOVE_POS(p, num) { \ p->additionalOffset += (num); \ - p->matchFinder.Skip(p->matchFinderObj, (num)); } + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) @@ -954,7 +1013,8 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) const Byte *p2 = p1 + len; ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; const Byte *lim = p1 + numAvail; - for (; p2 != lim && *p2 == p2[dif]; p2++); + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} return (unsigned)(p2 - p1); } } @@ -977,7 +1037,7 @@ static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) + GET_PRICE_1(p->isRep[state]) \ + GET_PRICE_0(p->isRepG0[state]) - +MY_FORCE_INLINE static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; @@ -1011,14 +1071,14 @@ static unsigned Backward(CLzmaEnc *p, unsigned cur) for (;;) { UInt32 dist = p->opt[cur].dist; - UInt32 len = p->opt[cur].len; - UInt32 extra = p->opt[cur].extra; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; cur -= len; if (extra) { wr--; - p->opt[wr].len = len; + p->opt[wr].len = (UInt32)len; cur -= extra; len = extra; if (extra == 1) @@ -1045,7 +1105,7 @@ static unsigned Backward(CLzmaEnc *p, unsigned cur) wr--; p->opt[wr].dist = dist; - p->opt[wr].len = len; + p->opt[wr].len = (UInt32)len; } } @@ -1102,7 +1162,8 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) repLens[i] = 0; continue; } - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} repLens[i] = len; if (len > repLens[repMaxIndex]) repMaxIndex = i; @@ -1111,7 +1172,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (repLens[repMaxIndex] >= p->numFastBytes) { unsigned len; - p->backRes = repMaxIndex; + p->backRes = (UInt32)repMaxIndex; len = repLens[repMaxIndex]; MOVE_POS(p, len - 1) return len; @@ -1128,8 +1189,12 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) curByte = *data; matchByte = *(data - reps[0]); + + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) + if (last < 2 && curByte != matchByte) { p->backRes = MARK_LIT; return 1; @@ -1146,13 +1211,14 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : LitEnc_GetPrice(probs, curByte, p->ProbPrices)); } - + MakeAs_Lit(&p->opt[1]); matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - if (matchByte == curByte) + // 18.06 + if (matchByte == curByte && repLens[0] == 0) { UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); if (shortRepPrice < p->opt[1].price) @@ -1160,16 +1226,13 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) p->opt[1].price = shortRepPrice; MakeAs_ShortRep(&p->opt[1]); } + if (last < 2) + { + p->backRes = p->opt[1].dist; + return 1; + } } - - last = (mainLen >= repLens[repMaxIndex] ? mainLen : repLens[repMaxIndex]); - - if (last < 2) - { - p->backRes = p->opt[1].dist; - return 1; - } - + p->opt[1].len = 1; p->opt[0].reps[0] = reps[0]; @@ -1177,13 +1240,6 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) p->opt[0].reps[2] = reps[2]; p->opt[0].reps[3] = reps[3]; - { - unsigned len = last; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - } - // ---------- REP ---------- for (i = 0; i < LZMA_NUM_REPS; i++) @@ -1195,13 +1251,13 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); do { - UInt32 price2 = price + p->repLenEnc.prices[posState][(size_t)repLen - 2]; + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); COptimal *opt = &p->opt[repLen]; if (price2 < opt->price) { opt->price = price2; - opt->len = repLen; - opt->dist = i; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; opt->extra = 0; } } @@ -1211,38 +1267,41 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) // ---------- MATCH ---------- { - unsigned len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); + unsigned len = repLens[0] + 1; if (len <= mainLen) { unsigned offs = 0; UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); - while (len > matches[offs]) - offs += 2; + if (len < 2) + len = 2; + else + while (len > matches[offs]) + offs += 2; for (; ; len++) { COptimal *opt; UInt32 dist = matches[(size_t)offs + 1]; - UInt32 price2 = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); unsigned lenToPosState = GetLenToPosState(len); if (dist < kNumFullDistances) - price2 += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; else { unsigned slot; GetPosSlot2(dist, slot); - price2 += p->alignPrices[dist & kAlignMask]; - price2 += p->posSlotPrices[lenToPosState][slot]; + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; } opt = &p->opt[len]; - if (price2 < opt->price) + if (price < opt->price) { - opt->price = price2; - opt->len = len; + opt->price = price; + opt->len = (UInt32)len; opt->dist = dist + LZMA_NUM_REPS; opt->extra = 0; } @@ -1277,16 +1336,43 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) for (;;) { - UInt32 numAvail, numAvailFull; + unsigned numAvail; + UInt32 numAvailFull; unsigned newLen, numPairs, prev, state, posState, startLen; - UInt32 curPrice, litPrice, matchPrice, repMatchPrice; - Bool nextIsLit; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; Byte curByte, matchByte; const Byte *data; COptimal *curOpt, *nextOpt; if (++cur == last) - return Backward(p, cur); + break; + + // 18.06 + if (cur >= kNumOpts - 64) + { + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) + { + price = price2; + best = j; + } + } + { + unsigned delta = best - cur; + if (delta != 0) + { + MOVE_POS(p, delta); + } + } + cur = best; + break; + } newLen = ReadMatchDistances(p, &numPairs); @@ -1294,15 +1380,24 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { p->numPairs = numPairs; p->longestMatchLen = newLen; - return Backward(p, cur); + break; } curOpt = &p->opt[cur]; + + position++; + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + prev = cur - curOpt->len; - + if (curOpt->len == 1) { - state = p->opt[prev].state; + state = (unsigned)p->opt[prev].state; if (IsShortRep(curOpt)) state = kShortRepNextStates[state]; else @@ -1316,14 +1411,14 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (curOpt->extra) { - prev -= curOpt->extra; + prev -= (unsigned)curOpt->extra; state = kState_RepAfterLit; if (curOpt->extra == 1) - state = (dist < LZMA_NUM_REPS) ? kState_RepAfterLit : kState_MatchAfterLit; + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); } else { - state = p->opt[prev].state; + state = (unsigned)p->opt[prev].state; if (dist < LZMA_NUM_REPS) state = kRepNextStates[state]; else @@ -1379,7 +1474,6 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) curByte = *data; matchByte = *(data - reps[0]); - position++; posState = (position & p->pbMask); /* @@ -1391,13 +1485,25 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) < MATCH [ : LIT : REP_0 ] */ - curPrice = curOpt->price; - litPrice = curPrice + GET_PRICE_0(p->isMatch[state][posState]); + { + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); + } nextOpt = &p->opt[(size_t)cur + 1]; nextIsLit = False; - // if (litPrice >= nextOpt->price) litPrice = 0; else // 18.new + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else { const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); litPrice += (!IsLitState(state) ? @@ -1413,21 +1519,32 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } } - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); + numAvailFull = p->numAvail; + { + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; + } + + // 18.06 // ---------- SHORT_REP ---------- - // if (IsLitState(state)) // 18.new + if (IsLitState(state)) // 18.new if (matchByte == curByte) - // if (repMatchPrice < nextOpt->price) // 18.new - if (nextOpt->len < 2 + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt || (nextOpt->dist != 0 - && nextOpt->extra <= 1 // 17.old - )) + // && nextOpt->extra <= 1 // 17.old + ) + ) { UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); - if (shortRepPrice <= nextOpt->price) // 17.old - // if (shortRepPrice < nextOpt->price) // 18.new + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new { nextOpt->price = shortRepPrice; nextOpt->len = 1; @@ -1436,13 +1553,6 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } } - numAvailFull = p->numAvail; - { - UInt32 temp = kNumOpts - 1 - cur; - if (numAvailFull > temp) - numAvailFull = temp; - } - if (numAvailFull < 2) continue; numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); @@ -1451,9 +1561,8 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) // ---------- LIT : REP_0 ---------- - if ( - // litPrice != 0 && // 18.new - !nextIsLit + if (!nextIsLit + && litPrice != 0 // 18.new && matchByte != curByte && numAvailFull > 2) { @@ -1464,7 +1573,8 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) unsigned limit = p->numFastBytes + 1; if (limit > numAvailFull) limit = numAvailFull; - for (len = 3; len < limit && data[len] == data2[len]; len++); + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} { unsigned state2 = kLiteralNextStates[state]; @@ -1472,8 +1582,9 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); { unsigned offset = cur + len; - while (last < offset) - p->opt[++last].price = kInfinityPrice; + + if (last < offset) + last = offset; // do { @@ -1481,14 +1592,14 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) COptimal *opt; len--; // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); - price2 = price + p->repLenEnc.prices[posState2][len - LZMA_MATCH_LEN_MIN]; + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); opt = &p->opt[offset]; // offset--; if (price2 < opt->price) { opt->price = price2; - opt->len = len; + opt->len = (UInt32)len; opt->dist = 0; opt->extra = 1; } @@ -1500,6 +1611,7 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } startLen = 2; /* speed optimization */ + { // ---------- REP ---------- unsigned repIndex = 0; // 17.old @@ -1512,24 +1624,28 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (data[0] != data2[0] || data[1] != data2[1]) continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} // if (len < startLen) continue; // 18.new: speed optimization - while (last < cur + len) - p->opt[++last].price = kInfinityPrice; + { + unsigned offset = cur + len; + if (last < offset) + last = offset; + } { unsigned len2 = len; price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); do { - UInt32 price2 = price + p->repLenEnc.prices[posState][(size_t)len2 - 2]; + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); COptimal *opt = &p->opt[cur + len2]; if (price2 < opt->price) { opt->price = price2; - opt->len = len2; - opt->dist = repIndex; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; opt->extra = 0; } } @@ -1549,15 +1665,14 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (limit > numAvailFull) limit = numAvailFull; - for (; len2 < limit && data[len2] == data2[len2]; len2++); - - len2 -= len; - if (len2 >= 3) + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { unsigned state2 = kRepNextStates[state]; unsigned posState2 = (position + len) & p->pbMask; - price += - p->repLenEnc.prices[posState][(size_t)len - 2] + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + GET_PRICE_0(p->isMatch[state2][posState2]) + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), data[len], data2[len], p->ProbPrices); @@ -1568,31 +1683,40 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { { unsigned offset = cur + len + len2; - while (last < offset) - p->opt[++last].price = kInfinityPrice; + + if (last < offset) + last = offset; // do { - unsigned price2; + UInt32 price2; COptimal *opt; len2--; // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); - price2 = price + p->repLenEnc.prices[posState2][len2 - LZMA_MATCH_LEN_MIN]; + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); opt = &p->opt[offset]; // offset--; if (price2 < opt->price) { opt->price = price2; - opt->len = len2; + opt->len = (UInt32)len2; opt->extra = (CExtra)(len + 1); - opt->dist = repIndex; + opt->dist = (UInt32)repIndex; } } // while (len2 >= 3); } } + } } } } @@ -1604,17 +1728,23 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { newLen = numAvail; for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; + matches[numPairs] = (UInt32)newLen; numPairs += 2; } + // startLen = 2; /* speed optimization */ + if (newLen >= startLen) { UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); UInt32 dist; unsigned offs, posSlot, len; - while (last < cur + newLen) - p->opt[++last].price = kInfinityPrice; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } offs = 0; while (startLen > matches[offs]) @@ -1626,27 +1756,29 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) for (len = /*2*/ startLen; ; len++) { - UInt32 price = normalMatchPrice + p->lenEnc.prices[posState][(size_t)len - LZMA_MATCH_LEN_MIN]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); { COptimal *opt; - unsigned lenToPosState = len - 2; lenToPosState = GetLenToPosState2(lenToPosState); + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); if (dist < kNumFullDistances) - price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; else - price += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; opt = &p->opt[cur + len]; if (price < opt->price) { opt->price = price; - opt->len = len; + opt->len = (UInt32)len; opt->dist = dist + LZMA_NUM_REPS; opt->extra = 0; } } - if (/*_maxMode && */ len == matches[offs]) + if (len == matches[offs]) { + // if (p->_maxMode) { // MATCH : LIT : REP_0 const Byte *data2 = data - dist - 1; @@ -1655,11 +1787,17 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) if (limit > numAvailFull) limit = numAvailFull; - for (; len2 < limit && data[len2] == data2[len2]; len2++); + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) + { + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} len2 -= len; - if (len2 >= 3) + // if (len2 >= 3) { unsigned state2 = kMatchNextStates[state]; unsigned posState2 = (position + len) & p->pbMask; @@ -1675,27 +1813,30 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) price += GetPrice_Rep_0(p, state2, posState2); offset = cur + len + len2; - while (last < offset) - p->opt[++last].price = kInfinityPrice; + + if (last < offset) + last = offset; // do { UInt32 price2; COptimal *opt; len2--; // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); - price2 = price + p->repLenEnc.prices[posState2][len2 - LZMA_MATCH_LEN_MIN]; + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); opt = &p->opt[offset]; // offset--; if (price2 < opt->price) { opt->price = price2; - opt->len = len2; + opt->len = (UInt32)len2; opt->extra = (CExtra)(len + 1); opt->dist = dist + LZMA_NUM_REPS; } } // while (len2 >= 3); } + + } offs += 2; if (offs == numPairs) @@ -1707,6 +1848,12 @@ static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) } } } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); } @@ -1733,6 +1880,7 @@ static unsigned GetOptimumFast(CLzmaEnc *p) p->backRes = MARK_LIT; if (numAvail < 2) return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; @@ -1744,10 +1892,11 @@ static unsigned GetOptimumFast(CLzmaEnc *p) const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} if (len >= p->numFastBytes) { - p->backRes = i; + p->backRes = (UInt32)i; MOVE_POS(p, len - 1) return len; } @@ -1791,7 +1940,7 @@ static unsigned GetOptimumFast(CLzmaEnc *p) || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) { - p->backRes = repIndex; + p->backRes = (UInt32)repIndex; MOVE_POS(p, repLen - 1) return repLen; } @@ -1930,23 +2079,22 @@ MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) } - -static void FillAlignPrices(CLzmaEnc *p) +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { unsigned i; const CProbPrice *ProbPrices = p->ProbPrices; const CLzmaProb *probs = p->posAlignEncoder; - p->alignPriceCount = 0; + // p->alignPriceCount = 0; for (i = 0; i < kAlignTableSize / 2; i++) { UInt32 price = 0; - unsigned symbol = i; + unsigned sym = i; unsigned m = 1; unsigned bit; UInt32 prob; - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; prob = probs[m]; p->alignPrices[i ] = price + GET_PRICEa_0(prob); p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); @@ -1955,78 +2103,97 @@ static void FillAlignPrices(CLzmaEnc *p) } -static void FillDistancesPrices(CLzmaEnc *p) +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) { + // int y; for (y = 0; y < 100; y++) { + UInt32 tempPrices[kNumFullDistances]; - unsigned i, lenToPosState; + unsigned i, lps; const CProbPrice *ProbPrices = p->ProbPrices; p->matchPriceCount = 0; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) { unsigned posSlot = GetPosSlot1(i); - unsigned footerBits = ((posSlot >> 1) - 1); + unsigned footerBits = (posSlot >> 1) - 1; unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); - - const CLzmaProb *probs = p->posEncoders + base; UInt32 price = 0; unsigned m = 1; - unsigned symbol = i - base; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) do { - unsigned bit = symbol & 1; - symbol >>= 1; + unsigned bit = sym & 1; + sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; } while (--footerBits); - tempPrices[i] = price; + + { + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); + } } - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) + for (lps = 0; lps < kNumLenToPosStates; lps++) { - unsigned posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - unsigned distTableSize = p->distTableSize; - const CLzmaProb *probs = encoder; - for (posSlot = 0; posSlot < distTableSize; posSlot += 2) + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) { - // posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - UInt32 price = 0; + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; unsigned bit; - unsigned symbol = (posSlot >> 1) + (1 << (kNumPosSlotBits - 1)); - UInt32 prob; - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); - bit = symbol & 1; symbol >>= 1; price += GET_PRICEa(probs[symbol], bit); - prob = probs[(posSlot >> 1) + (1 << (kNumPosSlotBits - 1))]; - posSlotPrices[posSlot ] = price + GET_PRICEa_0(prob); - posSlotPrices[posSlot + 1] = price + GET_PRICEa_1(prob); + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } } - for (posSlot = kEndPosModelIndex; posSlot < distTableSize; posSlot++) - posSlotPrices[posSlot] += ((UInt32)(((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - { - distancesPrices[0] = posSlotPrices[0]; - distancesPrices[1] = posSlotPrices[1]; - distancesPrices[2] = posSlotPrices[2]; - distancesPrices[3] = posSlotPrices[3]; - } + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + for (i = 4; i < kNumFullDistances; i += 2) { UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; - distancesPrices[i ] = slotPrice + tempPrices[i]; - distancesPrices[i + 1] = slotPrice + tempPrices[i + 1]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; } } } + // } } @@ -2243,10 +2410,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa if (len != 1) { LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); - if (!p->fastMode) - if (--p->repLenEnc.counters[posState] == 0) - LenPriceEnc_UpdateTable(&p->repLenEnc, posState, &p->repLenProbs, p->ProbPrices); - + --p->repLenEncCounter; p->state = kRepNextStates[p->state]; } } @@ -2258,9 +2422,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa p->state = kMatchNextStates[p->state]; LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); - if (!p->fastMode) - if (--p->lenEnc.counters[posState] == 0) - LenPriceEnc_UpdateTable(&p->lenEnc, posState, &p->lenProbs, p->ProbPrices); + // --p->lenEnc.counter; dist -= LZMA_NUM_REPS; p->reps[3] = p->reps[2]; @@ -2272,17 +2434,17 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa GetPosSlot(dist, posSlot); // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); { - UInt32 symbol = posSlot + (1 << kNumPosSlotBits); + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); range = p->rc.range; probs = p->posSlotEncoder[GetLenToPosState(len)]; do { - CLzmaProb *prob = probs + (symbol >> kNumPosSlotBits); - UInt32 bit = (symbol >> (kNumPosSlotBits - 1)) & 1; - symbol <<= 1; + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; RC_BIT(&p->rc, prob, bit); } - while (symbol < (1 << kNumPosSlotBits * 2)); + while (sym < (1 << kNumPosSlotBits * 2)); p->rc.range = range; } @@ -2293,7 +2455,7 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa if (dist < kNumFullDistances) { unsigned base = ((2 | (posSlot & 1)) << footerBits); - RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, dist - base); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); } else { @@ -2329,14 +2491,14 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); p->rc.range = range; - p->alignPriceCount++; + // p->alignPriceCount++; } } } } } - nowPos32 += len; + nowPos32 += (UInt32)len; p->additionalOffset -= len; if (p->additionalOffset == 0) @@ -2345,10 +2507,27 @@ static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpa if (!p->fastMode) { - if (p->matchPriceCount >= (1 << 7)) - FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) + /* + if (p->alignPriceCount >= 16) // kAlignTableSize FillAlignPrices(p); + if (p->matchPriceCount >= 128) + FillDistancesPrices(p); + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { + FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + } } if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) @@ -2488,12 +2667,19 @@ void LzmaEnc_Init(CLzmaEnc *p) p->optEnd = 0; p->optCur = 0; + + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } + p->additionalOffset = 0; p->pbMask = (1 << p->pb) - 1; p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); } + void LzmaEnc_InitPrices(CLzmaEnc *p) { if (!p->fastMode) @@ -2505,6 +2691,9 @@ void LzmaEnc_InitPrices(CLzmaEnc *p) p->lenEnc.tableSize = p->repLenEnc.tableSize = p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; + + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); } @@ -2581,7 +2770,7 @@ typedef struct ISeqOutStream vt; Byte *data; SizeT rem; - Bool overflow; + BoolInt overflow; } CLzmaEnc_SeqOutStreamBuf; static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) @@ -2613,7 +2802,7 @@ const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp) } -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { CLzmaEnc *p = (CLzmaEnc *)pp; diff --git a/C/MtCoder.c b/C/MtCoder.c index ddc7c028..5667f2d5 100644 --- a/C/MtCoder.c +++ b/C/MtCoder.c @@ -1,5 +1,5 @@ /* MtCoder.c -- Multi-thread Coder -2018-02-21 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -119,7 +119,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) unsigned bi; SRes res; SRes res2; - Bool finished; + BoolInt finished; unsigned bufIndex; size_t size; const Byte *inData; @@ -294,7 +294,7 @@ static SRes ThreadFunc2(CMtCoderThread *t) if (++wi >= mtc->numBlocksMax) wi = 0; { - Bool isReady; + BoolInt isReady; CriticalSection_Enter(&mtc->cs); @@ -547,7 +547,7 @@ SRes MtCoder_Code(CMtCoder *p) { const CMtCoderBlock *block = &p->blocks[bi]; unsigned bufIndex = block->bufIndex; - Bool finished = block->finished; + BoolInt finished = block->finished; if (res == SZ_OK && block->res != SZ_OK) res = block->res; diff --git a/C/MtCoder.h b/C/MtCoder.h index 7982e847..603329d3 100644 --- a/C/MtCoder.h +++ b/C/MtCoder.h @@ -1,5 +1,5 @@ /* MtCoder.h -- Multi-thread Coder -2018-02-21 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H @@ -67,7 +67,7 @@ typedef struct { SRes res; unsigned bufIndex; - Bool finished; + BoolInt finished; } CMtCoderBlock; @@ -97,7 +97,7 @@ typedef struct _CMtCoder CAutoResetEvent readEvent; CSemaphore blocksSemaphore; - Bool stopReading; + BoolInt stopReading; SRes readRes; #ifdef MTCODER__USE_WRITE_THREAD diff --git a/C/MtDec.c b/C/MtDec.c index 60d31b07..25a8b046 100644 --- a/C/MtDec.c +++ b/C/MtDec.c @@ -1,5 +1,5 @@ /* MtDec.c -- Multi-thread Decoder -2018-03-02 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -88,12 +88,13 @@ static WRes ArEvent_OptCreate_And_Reset(CEvent *p) } - -typedef struct +struct __CMtDecBufLink { - void *next; + struct __CMtDecBufLink *next; void *pad[3]; -} CMtDecBufLink; +}; + +typedef struct __CMtDecBufLink CMtDecBufLink; #define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) #define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) @@ -197,7 +198,7 @@ static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) } -static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, Bool *wasInterrupted) +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted) { SRes res; CriticalSection_Enter(&p->mtProgress.cs); @@ -207,7 +208,7 @@ static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, Bool *wasInter return res; } -static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, Bool *wasInterrupted) +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, BoolInt *wasInterrupted) { SRes res; CriticalSection_Enter(&p->mtProgress.cs); @@ -271,9 +272,9 @@ static WRes ThreadFunc2(CMtDecThread *t) for (;;) { SRes res, codeRes; - Bool wasInterrupted, isAllocError, overflow, finish; + BoolInt wasInterrupted, isAllocError, overflow, finish; SRes threadingErrorSRes; - Bool needCode, needWrite, needContinue; + BoolInt needCode, needWrite, needContinue; size_t inDataSize_Start; UInt64 inDataSize; @@ -289,7 +290,7 @@ static WRes ThreadFunc2(CMtDecThread *t) Byte *afterEndData = NULL; size_t afterEndData_Size = 0; - Bool canCreateNewThread = False; + BoolInt canCreateNewThread = False; // CMtDecCallbackInfo parse; CMtDecThread *nextThread; @@ -629,7 +630,7 @@ static WRes ThreadFunc2(CMtDecThread *t) if (res == SZ_OK && needCode && codeRes == SZ_OK) { - Bool isStartBlock = True; + BoolInt isStartBlock = True; CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; for (;;) @@ -691,9 +692,9 @@ static WRes ThreadFunc2(CMtDecThread *t) RINOK_THREAD(Event_Wait(&t->canWrite)); { - Bool isErrorMode = False; - Bool canRecode = True; - Bool needWriteToStream = needWrite; + BoolInt isErrorMode = False; + BoolInt canRecode = True; + BoolInt needWriteToStream = needWrite; if (p->exitThread) return 0; // it's never executed in normal cases diff --git a/C/MtDec.h b/C/MtDec.h index b445bc9d..9864cc87 100644 --- a/C/MtDec.h +++ b/C/MtDec.h @@ -1,5 +1,5 @@ /* MtDec.h -- Multi-thread Decoder -2018-03-02 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __MT_DEC_H #define __MT_DEC_H @@ -76,7 +76,7 @@ typedef struct // out EMtDecParseState state; - Bool canCreateNewThread; + BoolInt canCreateNewThread; UInt64 outPos; // check it (size_t) } CMtDecCallbackInfo; @@ -107,11 +107,11 @@ typedef struct if (*canRecode), we didn't flush current block data, so we still can decode current block later. */ SRes (*Write)(void *p, unsigned coderIndex, - Bool needWriteToStream, + BoolInt needWriteToStream, const Byte *src, size_t srcSize, // int srcFinished, - Bool *needContinue, - Bool *canRecode); + BoolInt *needContinue, + BoolInt *canRecode); } IMtDecCallback; @@ -140,22 +140,22 @@ typedef struct _CMtDec size_t allocatedBufsSize; - Bool exitThread; + BoolInt exitThread; WRes exitThreadWRes; UInt64 blockIndex; - Bool isAllocError; - Bool overflow; + BoolInt isAllocError; + BoolInt overflow; SRes threadingErrorSRes; - Bool needContinue; + BoolInt needContinue; // CAutoResetEvent finishedEvent; SRes readRes; SRes codeRes; - Bool wasInterrupted; + BoolInt wasInterrupted; unsigned numStartedThreads_Limit; unsigned numStartedThreads; @@ -164,14 +164,14 @@ typedef struct _CMtDec size_t crossStart; size_t crossEnd; UInt64 readProcessed; - Bool readWasFinished; + BoolInt readWasFinished; UInt64 inProcessed; unsigned filledThreadStart; unsigned numFilledThreads; #ifndef _7ZIP_ST - Bool needInterrupt; + BoolInt needInterrupt; UInt64 interruptIndex; CMtProgress mtProgress; CMtDecThread threads[MTDEC__THREADS_MAX]; diff --git a/C/Ppmd7.c b/C/Ppmd7.c index ef93cb2d..80e7de9a 100644 --- a/C/Ppmd7.c +++ b/C/Ppmd7.c @@ -1,5 +1,5 @@ /* Ppmd7.c -- PPMdH codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -95,7 +95,7 @@ void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) p->Base = 0; } -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) { if (!p->Base || p->Size != size) { @@ -342,7 +342,7 @@ void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder) p->DummySee.Count = 64; /* unused */ } -static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) +static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) { CPpmd_State upState; CTX_PTR c = p->MinContext; diff --git a/C/Ppmd7.h b/C/Ppmd7.h index ee2c035d..cce93f12 100644 --- a/C/Ppmd7.h +++ b/C/Ppmd7.h @@ -1,5 +1,5 @@ /* Ppmd7.h -- PPMdH compression codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ /* This code supports virtual RangeDecoder and includes the implementation @@ -60,7 +60,7 @@ typedef struct } CPpmd7; void Ppmd7_Construct(CPpmd7 *p); -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL) @@ -115,7 +115,7 @@ typedef struct } CPpmd7z_RangeDec; void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); #define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); diff --git a/C/Ppmd7Dec.c b/C/Ppmd7Dec.c index 3023b677..20264071 100644 --- a/C/Ppmd7Dec.c +++ b/C/Ppmd7Dec.c @@ -1,5 +1,5 @@ /* Ppmd7Dec.c -- PPMdH Decoder -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -8,7 +8,7 @@ This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #define kTopValue (1 << 24) -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) { unsigned i; p->Code = 0; diff --git a/C/Ppmd8.c b/C/Ppmd8.c index 3058ada1..3a6ee6c4 100644 --- a/C/Ppmd8.c +++ b/C/Ppmd8.c @@ -1,5 +1,5 @@ /* Ppmd8.c -- PPMdI codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.I (2002): Dmitry Shkarin : Public domain */ #include "Precomp.h" @@ -93,7 +93,7 @@ void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc) p->Base = 0; } -Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) +BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc) { if (!p->Base || p->Size != size) { @@ -605,7 +605,7 @@ static void RestoreModel(CPpmd8 *p, CTX_PTR c1 } } -static CTX_PTR CreateSuccessors(CPpmd8 *p, Bool skip, CPpmd_State *s1, CTX_PTR c) +static CTX_PTR CreateSuccessors(CPpmd8 *p, BoolInt skip, CPpmd_State *s1, CTX_PTR c) { CPpmd_State upState; Byte flags; diff --git a/C/Ppmd8.h b/C/Ppmd8.h index 8f581c06..ecaa8bac 100644 --- a/C/Ppmd8.h +++ b/C/Ppmd8.h @@ -1,5 +1,5 @@ /* Ppmd8.h -- PPMdI codec -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -86,7 +86,7 @@ typedef struct } CPpmd8; void Ppmd8_Construct(CPpmd8 *p); -Bool Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc); +BoolInt Ppmd8_Alloc(CPpmd8 *p, UInt32 size, ISzAllocPtr alloc); void Ppmd8_Free(CPpmd8 *p, ISzAllocPtr alloc); void Ppmd8_Init(CPpmd8 *p, unsigned maxOrder, unsigned restoreMethod); #define Ppmd8_WasAllocated(p) ((p)->Base != NULL) @@ -121,7 +121,7 @@ CPpmd_See *Ppmd8_MakeEscFreq(CPpmd8 *p, unsigned numMasked, UInt32 *scale); /* ---------- Decode ---------- */ -Bool Ppmd8_RangeDec_Init(CPpmd8 *p); +BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p); #define Ppmd8_RangeDec_IsFinishedOK(p) ((p)->Code == 0) int Ppmd8_DecodeSymbol(CPpmd8 *p); /* returns: -1 as EndMarker, -2 as DataError */ diff --git a/C/Ppmd8Dec.c b/C/Ppmd8Dec.c index fefc17bb..05c78c50 100644 --- a/C/Ppmd8Dec.c +++ b/C/Ppmd8Dec.c @@ -1,5 +1,5 @@ /* Ppmd8Dec.c -- PPMdI Decoder -2017-04-03 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on: PPMd var.I (2002): Dmitry Shkarin : Public domain Carryless rangecoder (1999): Dmitry Subbotin : Public domain */ @@ -11,7 +11,7 @@ This code is based on: #define kTop (1 << 24) #define kBot (1 << 15) -Bool Ppmd8_RangeDec_Init(CPpmd8 *p) +BoolInt Ppmd8_RangeDec_Init(CPpmd8 *p) { unsigned i; p->Low = 0; diff --git a/C/Util/7z/7zMain.c b/C/Util/7z/7zMain.c index 82aac89e..1c02b48e 100644 --- a/C/Util/7z/7zMain.c +++ b/C/Util/7z/7zMain.c @@ -1,5 +1,5 @@ /* 7zMain.c - Test application for 7z Decoder -2018-04-19 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -176,7 +176,7 @@ static SRes Utf16_To_Char(CBuf *buf, const UInt16 *s char defaultChar = '_'; BOOL defUsed; unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, s, len, (char *)buf->data, size, &defaultChar, &defUsed); + numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); if (numChars == 0 || numChars >= size) return SZ_ERROR_FAIL; buf->data[numChars] = 0; @@ -202,7 +202,7 @@ static WRes MyCreateDir(const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return CreateDirectoryW(name, NULL) ? 0 : GetLastError(); + return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); #else @@ -227,7 +227,7 @@ static WRes MyCreateDir(const UInt16 *name) static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, name); + return OutFile_OpenW(p, (LPCWSTR)name); #else CBuf buf; WRes res; @@ -354,7 +354,7 @@ static void PrintError(char *s) PrintLF(); } -static void GetAttribString(UInt32 wa, Bool isDir, char *s) +static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) { #ifdef USE_WINDOWS_FILE s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.'); @@ -430,7 +430,7 @@ int MY_CDECL main(int numargs, char *args[]) res = SZ_OK; { - lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); if (!lookStream.buf) res = SZ_ERROR_MEM; else @@ -647,7 +647,7 @@ int MY_CDECL main(int numargs, char *args[]) We remove posix bits, if we detect posix mode field */ if ((attrib & 0xF0000000) != 0) attrib &= 0x7FFF; - SetFileAttributesW(destPath, attrib); + SetFileAttributesW((LPCWSTR)destPath, attrib); } #endif } diff --git a/C/Util/7zipInstall/7zipInstall.c b/C/Util/7zipInstall/7zipInstall.c index 519546e9..bc0e3c6c 100644 --- a/C/Util/7zipInstall/7zipInstall.c +++ b/C/Util/7zipInstall/7zipInstall.c @@ -1,5 +1,5 @@ /* 7zipInstall.c - 7-Zip Installer -2017-08-28 : Igor Pavlov : Public domain */ +2019-02-19 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -82,16 +82,16 @@ static LPCWSTR const k_Reg_Path32 = L"Path" #define k_Reg_WOW_Flag_32 0 #endif -#define k_7zip_CLSID L"{23170F69-0803-278A-1000-000100020001}" +#define k_7zip_CLSID L"{23170F69-20BB-278A-1000-000100020000}" static LPCWSTR const k_Reg_CLSID_7zip = L"CLSID\\" k_7zip_CLSID; static LPCWSTR const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\InprocServer32"; #define g_AllUsers True -static Bool g_Install_was_Pressed; -static Bool g_Finished; -static Bool g_SilentMode; +static BoolInt g_Install_was_Pressed; +static BoolInt g_Finished; +static BoolInt g_SilentMode; static HWND g_HWND; static HWND g_Path_HWND; @@ -121,13 +121,51 @@ static void PrintErrorMessage(const char *s) } +typedef DWORD (WINAPI * Func_GetFileVersionInfoSizeW)(LPCWSTR lptstrFilename, LPDWORD lpdwHandle); +typedef BOOL (WINAPI * Func_GetFileVersionInfoW)(LPCWSTR lptstrFilename, DWORD dwHandle, DWORD dwLen, LPVOID lpData); +typedef BOOL (WINAPI * Func_VerQueryValueW)(const LPVOID pBlock, LPWSTR lpSubBlock, LPVOID * lplpBuffer, PUINT puLen); + +static HMODULE g_version_dll_hModule; + static DWORD GetFileVersion(LPCWSTR s) { DWORD size = 0; - BYTE *vi = NULL; + void *vi = NULL; DWORD version = 0; + + Func_GetFileVersionInfoSizeW my_GetFileVersionInfoSizeW; + Func_GetFileVersionInfoW my_GetFileVersionInfoW; + Func_VerQueryValueW my_VerQueryValueW; + + if (!g_version_dll_hModule) + { + wchar_t buf[MAX_PATH + 100]; + { + unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (len == 0 || len > MAX_PATH) + return 0; + } + { + unsigned pos = (unsigned)lstrlenW(buf); + if (buf[pos - 1] != '\\') + buf[pos++] = '\\'; + lstrcpyW(buf + pos, L"version.dll"); + } + g_version_dll_hModule = LoadLibraryW(buf); + if (!g_version_dll_hModule) + return 0; + } + + my_GetFileVersionInfoSizeW = (Func_GetFileVersionInfoSizeW)GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoSizeW"); + my_GetFileVersionInfoW = (Func_GetFileVersionInfoW)GetProcAddress(g_version_dll_hModule, "GetFileVersionInfoW"); + my_VerQueryValueW = (Func_VerQueryValueW)GetProcAddress(g_version_dll_hModule, "VerQueryValueW"); + + if (!my_GetFileVersionInfoSizeW + || !my_GetFileVersionInfoW + || !my_VerQueryValueW) + return 0; - size = GetFileVersionInfoSizeW(s, NULL); + size = my_GetFileVersionInfoSizeW(s, NULL); if (size == 0) return 0; @@ -135,11 +173,11 @@ static DWORD GetFileVersion(LPCWSTR s) if (!vi) return 0; - if (GetFileVersionInfoW(s, 0, size, vi)) + if (my_GetFileVersionInfoW(s, 0, size, vi)) { VS_FIXEDFILEINFO *fi = NULL; UINT fiLen = 0; - if (VerQueryValueW(vi, L"\\", (LPVOID *)&fi, &fiLen)) + if (my_VerQueryValueW(vi, L"\\", (LPVOID *)&fi, &fiLen)) version = fi->dwFileVersionMS; } @@ -270,7 +308,7 @@ static int MyRegistry_QueryString2(HKEY hKey, LPCWSTR keyName, LPCWSTR valName, if (res != ERROR_SUCCESS) return False; { - Bool res2 = MyRegistry_QueryString(key, valName, dest); + BoolInt res2 = MyRegistry_QueryString(key, valName, dest); RegCloseKey(key); return res2; } @@ -343,7 +381,7 @@ static LONG MyRegistry_CreateKeyAndVal_32(HKEY parentKey, LPCWSTR keyName, LPCWS #define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufSize]; size_t numPrevBytes = 0; @@ -431,7 +469,7 @@ int CALLBACK BrowseCallbackProc(HWND hwnd, UINT uMsg, LPARAM lp, LPARAM data) return 0; } -static Bool MyBrowseForFolder(HWND owner, LPCWSTR title, UINT ulFlags, +static BoolInt MyBrowseForFolder(HWND owner, LPCWSTR title, UINT ulFlags, LPCWSTR initialFolder, LPWSTR resultPath) { WCHAR displayName[MAX_PATH]; @@ -694,7 +732,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) for (; i < 3; i++) { - Bool isOK = True; + BoolInt isOK = True; WCHAR link[MAX_PATH + 40]; WCHAR destPath[MAX_PATH + 40]; @@ -874,7 +912,7 @@ static void WriteShellEx() static const wchar_t *GetCmdParam(const wchar_t *s) { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; s++) { wchar_t c = *s; @@ -988,7 +1026,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (path[0] == 0) { HKEY key = 0; - Bool ok = False; + BoolInt ok = False; LONG res = RegOpenKeyExW(HKEY_CURRENT_USER, k_Reg_Software_7zip, 0, KEY_READ | k_Reg_WOW_Flag, &key); if (res == ERROR_SUCCESS) { @@ -1101,9 +1139,9 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, } -static Bool GetErrorMessage(DWORD errorCode, WCHAR *message) +static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message) { - LPVOID msgBuf; + LPWSTR msgBuf; if (FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM @@ -1201,7 +1239,7 @@ if (res == SZ_OK) if (res == SZ_OK) { - lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); if (!lookStream.buf) res = SZ_ERROR_MEM; else @@ -1272,7 +1310,7 @@ if (res == SZ_OK) temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); if (!g_SilentMode) SetWindowTextW(g_InfoLine_HWND, temp); @@ -1313,7 +1351,7 @@ if (res == SZ_OK) } { - // Bool skipFile = False; + // BoolInt skipFile = False; wcscpy(origPath, path); diff --git a/C/Util/7zipInstall/makefile b/C/Util/7zipInstall/makefile index ac696403..c4338d52 100644 --- a/C/Util/7zipInstall/makefile +++ b/C/Util/7zipInstall/makefile @@ -1,8 +1,6 @@ PROG = 7zipInstall.exe MY_FIXED = 1 -LIBS = $(LIBS) version.lib - !IFDEF _64BIT_INSTALLER CFLAGS = $(CFLAGS) -D_64BIT_INSTALLER !ENDIF diff --git a/C/Util/7zipUninstall/7zipUninstall.c b/C/Util/7zipUninstall/7zipUninstall.c index c3973817..06b14311 100644 --- a/C/Util/7zipUninstall/7zipUninstall.c +++ b/C/Util/7zipUninstall/7zipUninstall.c @@ -1,5 +1,5 @@ /* 7zipUninstall.c - 7-Zip Uninstaller -2018-03-01 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -62,7 +62,7 @@ static const WCHAR * const k_Reg_Path32 = L"Path" #define k_Reg_WOW_Flag_32 0 #endif -#define k_7zip_CLSID L"{23170F69-0803-278A-1000-000100020001}" +#define k_7zip_CLSID L"{23170F69-20BB-278A-1000-000100020000}" static const WCHAR * const k_Reg_CLSID_7zip = L"CLSID\\" k_7zip_CLSID; static const WCHAR * const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\\InprocServer32"; @@ -70,9 +70,9 @@ static const WCHAR * const k_Reg_CLSID_7zip_Inproc = L"CLSID\\" k_7zip_CLSID L"\ #define g_AllUsers True -static Bool g_Install_was_Pressed; -static Bool g_Finished; -static Bool g_SilentMode; +static BoolInt g_Install_was_Pressed; +static BoolInt g_Finished; +static BoolInt g_SilentMode; static HWND g_HWND; static HWND g_Path_HWND; @@ -94,7 +94,7 @@ static const WCHAR * const kUninstallExe = L"Uninstall.exe"; #define MAKE_CHAR_UPPER(c) ((((c) >= 'a' && (c) <= 'z') ? (c) -= 0x20 : (c))) -static Bool AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) +static BoolInt AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) { for (;;) { @@ -107,7 +107,7 @@ static Bool AreStringsEqual_NoCase(const wchar_t *s1, const wchar_t *s2) } } -static Bool IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) +static BoolInt IsString1PrefixedByString2_NoCase(const wchar_t *s1, const wchar_t *s2) { for (;;) { @@ -149,7 +149,7 @@ static int MyRegistry_QueryString2(HKEY hKey, LPCWSTR keyName, LPCWSTR valName, if (res != ERROR_SUCCESS) return False; { - Bool res2 = MyRegistry_QueryString(key, valName, dest); + BoolInt res2 = MyRegistry_QueryString(key, valName, dest); RegCloseKey(key); return res2; } @@ -180,7 +180,7 @@ static int MyRegistry_QueryString2_32(HKEY hKey, LPCWSTR keyName, LPCWSTR valNam if (res != ERROR_SUCCESS) return False; { - Bool res2 = MyRegistry_QueryString(key, valName, dest); + BoolInt res2 = MyRegistry_QueryString(key, valName, dest); RegCloseKey(key); return res2; } @@ -283,7 +283,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) for (; i < 3; i++) { - // Bool isOK = True; + // BoolInt isOK = True; WCHAR link[MAX_PATH + 40]; WCHAR destPath[MAX_PATH + 40]; @@ -300,7 +300,7 @@ static void SetShellProgramsGroup(HWND hwndOwner) { const size_t baseLen = wcslen(link); unsigned k; - Bool needDelete = False; + BoolInt needDelete = False; for (k = 0; k < 2; k++) { @@ -347,7 +347,7 @@ static const WCHAR * const k_AppPaths_7zFm = L"Software\\Microsoft\\Windows\\Cur static const WCHAR * const k_Uninstall_7zip = k_REG_Uninstall L"7-Zip-Zstandard"; -static Bool AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name) +static BoolInt AreEqual_Path_PrefixName(const wchar_t *s, const wchar_t *prefix, const wchar_t *name) { if (!IsString1PrefixedByString2_NoCase(s, prefix)) return False; @@ -444,7 +444,7 @@ static void WriteCLSID() static const wchar_t *GetCmdParam(const wchar_t *s) { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; s++) { wchar_t c = *s; @@ -469,7 +469,7 @@ static void RemoveQuotes(wchar_t *s) } } -static Bool DoesFileOrDirExist() +static BoolInt DoesFileOrDirExist() { return (GetFileAttributesW(path) != INVALID_FILE_ATTRIBUTES); } @@ -491,7 +491,7 @@ static BOOL RemoveFileAfterReboot() #define IS_LIMIT_CHAR(c) (c == 0 || c == ' ') -static Bool IsThereSpace(const wchar_t *s) +static BoolInt IsThereSpace(const wchar_t *s) { for (;;) { @@ -505,7 +505,7 @@ static Bool IsThereSpace(const wchar_t *s) static void AddPathParam(wchar_t *dest, const wchar_t *src) { - Bool needQuote = IsThereSpace(src); + BoolInt needQuote = IsThereSpace(src); if (needQuote) wcscat(dest, L"\""); wcscat(dest, src); @@ -515,9 +515,9 @@ static void AddPathParam(wchar_t *dest, const wchar_t *src) -static Bool GetErrorMessage(DWORD errorCode, WCHAR *message) +static BoolInt GetErrorMessage(DWORD errorCode, WCHAR *message) { - LPVOID msgBuf; + LPWSTR msgBuf; if (FormatMessageW( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM @@ -586,7 +586,7 @@ static int Install() SRes res = SZ_OK; WRes winRes = 0; - // Bool needReboot = False; + // BoolInt needReboot = False; const size_t pathLen = wcslen(path); if (!g_SilentMode) @@ -805,7 +805,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, lpCmdLine, int nCmdShow) { const wchar_t *cmdParams; - Bool useTemp = True; + BoolInt useTemp = True; UNUSED_VAR(hPrevInstance) UNUSED_VAR(lpCmdLine) @@ -1005,7 +1005,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (path[0] == 0) { HKEY key = 0; - Bool ok = False; + BoolInt ok = False; LONG res = RegOpenKeyExW(HKEY_CURRENT_USER, k_Reg_Software_7zip, 0, KEY_READ | k_Reg_WOW_Flag, &key); if (res == ERROR_SUCCESS) { diff --git a/C/Util/Lzma/LzmaUtil.c b/C/Util/Lzma/LzmaUtil.c index cf88c777..82130e85 100644 --- a/C/Util/Lzma/LzmaUtil.c +++ b/C/Util/Lzma/LzmaUtil.c @@ -1,5 +1,5 @@ /* LzmaUtil.c -- Test application for LZMA compression -2017-04-27 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "../../Precomp.h" @@ -177,7 +177,7 @@ static int main2(int numArgs, const char *args[], char *rs) char c; int res; int encodeMode; - Bool useOutFile = False; + BoolInt useOutFile = False; FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file); diff --git a/C/Util/SfxSetup/SfxSetup.c b/C/Util/SfxSetup/SfxSetup.c index bfbf4302..394369ab 100644 --- a/C/Util/SfxSetup/SfxSetup.c +++ b/C/Util/SfxSetup/SfxSetup.c @@ -1,5 +1,5 @@ /* SfxSetup.c - 7z SFX Setup -2017-04-04 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -127,7 +127,7 @@ static WRes MyCreateDir(const WCHAR *name) #define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufferSize]; size_t numPrevBytes = 0; @@ -163,7 +163,7 @@ static Bool FindSignature(CSzFile *stream, UInt64 *resPos) } } -static Bool DoesFileOrDirExist(const WCHAR *path) +static BoolInt DoesFileOrDirExist(const WCHAR *path) { WIN32_FIND_DATAW fd; HANDLE handle; @@ -254,7 +254,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, DWORD winRes; const wchar_t *cmdLineParams; const char *errorMessage = NULL; - Bool useShellExecute = True; + BoolInt useShellExecute = True; DWORD exitCode = 0; LoadSecurityDlls(); @@ -287,7 +287,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, cmdLineParams = GetCommandLineW(); #ifndef UNDER_CE { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; cmdLineParams++) { wchar_t c = *cmdLineParams; @@ -379,7 +379,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, if (res == SZ_OK) { - lookStream.buf = ISzAlloc_Alloc(&allocImp, kInputBufSize); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); if (!lookStream.buf) res = SZ_ERROR_MEM; else @@ -420,7 +420,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); { res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, @@ -527,7 +527,7 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, { WCHAR *temp = path + pathLen; UInt32 j; - SzArEx_GetFileNameUtf16(&db, executeFileIndex, temp); + SzArEx_GetFileNameUtf16(&db, executeFileIndex, (UInt16 *)temp); for (j = 0; temp[j] != 0; j++) if (temp[j] == '/') temp[j] = CHAR_PATH_SEPARATOR; diff --git a/C/Xz.h b/C/Xz.h index 7b88f516..fad56a3f 100644 --- a/C/Xz.h +++ b/C/Xz.h @@ -1,5 +1,5 @@ /* Xz.h - Xz interface -2018-02-28 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H @@ -53,7 +53,7 @@ typedef struct #define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); /* ---------- xz stream ---------- */ @@ -186,10 +186,10 @@ typedef struct Byte *outBuf; size_t outBufSize; size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) - Bool wasFinished; + BoolInt wasFinished; SRes res; ECoderStatus status; - // Bool SingleBufMode; + // BoolInt SingleBufMode; int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; @@ -241,9 +241,9 @@ typedef struct CXzCheck check; CSha256 sha; - Bool parseMode; - Bool headerParsedOk; - Bool decodeToStreamSignature; + BoolInt parseMode; + BoolInt headerParsedOk; + BoolInt decodeToStreamSignature; unsigned decodeOnlyOneBlock; Byte *outBuf; @@ -335,7 +335,7 @@ SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status); -Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, @@ -365,7 +365,7 @@ UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); */ void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); -Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p); +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); #define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) @@ -378,7 +378,7 @@ typedef struct { size_t inBufSize_ST; size_t outStep_ST; - Bool ignoreErrors; + BoolInt ignoreErrors; #ifndef _7ZIP_ST unsigned numThreads; diff --git a/C/XzDec.c b/C/XzDec.c index ebf19833..4f532720 100644 --- a/C/XzDec.c +++ b/C/XzDec.c @@ -1,5 +1,5 @@ /* XzDec.c -- Xz Decode -2018-04-24 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -245,7 +245,7 @@ SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocP CBraState *decoder; if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - decoder = p->p; + decoder = (CBraState *)p->p; if (!decoder) { decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); @@ -341,7 +341,7 @@ static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) typedef struct { CLzma2Dec decoder; - Bool outBufMode; + BoolInt outBufMode; } CLzma2Dec_Spec; @@ -400,7 +400,7 @@ static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *s res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder - *status = status2; + *status = (ECoderStatus)status2; return res; } @@ -637,8 +637,8 @@ static SRes MixCoder_Code(CMixCoder *p, for (;;) { - Bool processed = False; - Bool allFinished = True; + BoolInt processed = False; + BoolInt allFinished = True; SRes resMain = SZ_OK; unsigned i; @@ -761,7 +761,7 @@ SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; } -static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) +static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) { return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) && GetUi32(buf) == CrcCalc(buf + 4, 6) @@ -775,7 +775,7 @@ static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *b if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } -static Bool XzBlock_AreSupportedFilters(const CXzBlock *p) +static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) { unsigned numFilters = XzBlock_GetNumFilters(p) - 1; unsigned i; @@ -866,7 +866,7 @@ SRes XzBlock_Parse(CXzBlock *p, const Byte *header) static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; - Bool needReInit = True; + BoolInt needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) @@ -999,8 +999,8 @@ SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, SRes res; ECoderFinishMode finishMode2 = finishMode; - Bool srcFinished2 = srcFinished; - Bool destFinish = False; + BoolInt srcFinished2 = srcFinished; + BoolInt destFinish = False; if (p->block.packSize != (UInt64)(Int64)-1) { @@ -1346,12 +1346,12 @@ SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, } -Bool XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); } -Bool XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); } @@ -1423,18 +1423,18 @@ typedef struct size_t outCodeSize; ECoderStatus status; SRes codeRes; - Bool skipMode; - // Bool finishedWithMark; + BoolInt skipMode; + // BoolInt finishedWithMark; EMtDecParseState parseState; - Bool parsing_Truncated; - Bool atBlockHeader; + BoolInt parsing_Truncated; + BoolInt atBlockHeader; CXzStreamFlags streamFlags; // UInt64 numFinishedStreams UInt64 numStreams; UInt64 numTotalBlocks; UInt64 numBlocks; - Bool dec_created; + BoolInt dec_created; CXzUnpacker dec; Byte mtPad[1 << 7]; @@ -1458,14 +1458,14 @@ typedef struct ICompressProgress *progress; // CXzStatInfo *stat; - Bool finishMode; - Bool outSize_Defined; + BoolInt finishMode; + BoolInt outSize_Defined; UInt64 outSize; UInt64 outProcessed; UInt64 inProcessed; UInt64 readProcessed; - Bool readWasFinished; + BoolInt readWasFinished; SRes readRes; SRes writeRes; @@ -1473,14 +1473,14 @@ typedef struct size_t outBufSize; Byte *inBuf; size_t inBufSize; - Bool dec_created; + CXzUnpacker dec; ECoderStatus status; SRes codeRes; #ifndef _7ZIP_ST - Bool mainDecoderWasCalled; + BoolInt mainDecoderWasCalled; // int statErrorDefined; int finishedDecoderIndex; @@ -1494,12 +1494,12 @@ typedef struct // UInt64 numBadBlocks; SRes mainErrorCode; - Bool isBlockHeaderState_Parse; - Bool isBlockHeaderState_Write; + BoolInt isBlockHeaderState_Parse; + BoolInt isBlockHeaderState_Write; UInt64 outProcessed_Parse; - Bool parsing_Truncated; + BoolInt parsing_Truncated; - Bool mtc_WasConstructed; + BoolInt mtc_WasConstructed; CMtDec mtc; CXzDecMtThread coders[MTDEC__THREADS_MAX]; #endif @@ -1525,7 +1525,8 @@ CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) p->outBufSize = 0; p->inBuf = NULL; p->inBufSize = 0; - p->dec_created = False; + + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); p->unpackBlockMaxSize = 0; @@ -1573,11 +1574,7 @@ static void XzDecMt_FreeOutBufs(CXzDecMt *p) static void XzDecMt_FreeSt(CXzDecMt *p) { - if (p->dec_created) - { - XzUnpacker_Free(&p->dec); - p->dec_created = False; - } + XzUnpacker_Free(&p->dec); if (p->outBuf) { @@ -1968,11 +1965,11 @@ static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, #define XZDECMT_STREAM_WRITE_STEP (1 << 24) static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, - Bool needWriteToStream, + BoolInt needWriteToStream, const Byte *src, size_t srcSize, // int srcFinished, - Bool *needContinue, - Bool *canRecode) + BoolInt *needContinue, + BoolInt *canRecode) { CXzDecMt *me = (CXzDecMt *)pp; const CXzDecMtThread *coder = &me->coders[coderIndex]; @@ -2302,7 +2299,7 @@ void XzStatInfo_Clear(CXzStatInfo *p) static SRes XzDecMt_Decode_ST(CXzDecMt *p #ifndef _7ZIP_ST - , Bool tMode + , BoolInt tMode #endif , CXzStatInfo *stat) { @@ -2358,7 +2355,7 @@ static SRes XzDecMt_Decode_ST(CXzDecMt *p for (;;) { SizeT outSize; - Bool finished; + BoolInt finished; ECoderFinishMode finishMode; SizeT inProcessed; ECoderStatus status; @@ -2466,7 +2463,7 @@ static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, int finishMode, UInt64 readProcessed, UInt64 inProcessed, SRes res, ECoderStatus status, - Bool decodingTruncated, + BoolInt decodingTruncated, CXzStatInfo *stat) { UInt64 extraSize; @@ -2531,7 +2528,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, { CXzDecMt *p = (CXzDecMt *)pp; #ifndef _7ZIP_ST - Bool tMode; + BoolInt tMode; #endif XzStatInfo_Clear(stat); @@ -2564,13 +2561,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, p->codeRes = 0; p->status = CODER_STATUS_NOT_SPECIFIED; - if (!p->dec_created) - { - XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); - p->dec_created = True; - } XzUnpacker_Init(&p->dec); - *isMT = False; @@ -2600,6 +2591,8 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, { IMtDecCallback vt; + // we just free ST buffers here + // but we still keep state variables, that was set in XzUnpacker_Init() XzDecMt_FreeSt(p); p->outProcessed_Parse = 0; @@ -2636,7 +2629,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, vt.Write = XzDecMt_Callback_Write; { - Bool needContinue; + BoolInt needContinue; SRes res = MtDec_Code(&p->mtc); @@ -2665,7 +2658,7 @@ SRes XzDecMt_Decode(CXzDecMtHandle pp, if (!needContinue) { SRes codeRes; - Bool truncated = False; + BoolInt truncated = False; ECoderStatus status; CXzUnpacker *dec; diff --git a/C/XzEnc.c b/C/XzEnc.c index 432cbfe7..309eca94 100644 --- a/C/XzEnc.c +++ b/C/XzEnc.c @@ -1,5 +1,5 @@ /* XzEnc.c -- Xz Encode -2018-04-28 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -366,7 +366,7 @@ static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) SRes res; *size = sizeOriginal; res = p->StateCoder.Code2(p->StateCoder.p, - data, size, + (Byte *)data, size, p->buf + p->curPos, &srcLen, p->srcWasFinished, CODER_FINISH_ANY, &status); @@ -814,7 +814,7 @@ static SRes Xz_CompressBlock( SRes res; Byte *outBuf = NULL; size_t outSize = 0; - Bool useStream = (fp || inStream); + BoolInt useStream = (fp || inStream); // useStream = True; if (!useStream) @@ -940,7 +940,7 @@ typedef struct #ifndef _7ZIP_ST unsigned checkType; ISeqOutStream *outStream; - Bool mtCoder_WasConstructed; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; #endif diff --git a/C/XzIn.c b/C/XzIn.c index 42da1dec..792a6178 100644 --- a/C/XzIn.c +++ b/C/XzIn.c @@ -1,5 +1,5 @@ /* XzIn.c - Xz input -2018-02-02 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" @@ -28,7 +28,7 @@ SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes) +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; unsigned headerSize; diff --git a/C/brotli/Brotli-Adjust.sh b/C/brotli/Brotli-Adjust.sh index d2ca94f9..f720cac1 100755 --- a/C/brotli/Brotli-Adjust.sh +++ b/C/brotli/Brotli-Adjust.sh @@ -36,10 +36,13 @@ sed -i 's|include "../common|include "./common/|g' *.c for f in *.c; do mv $f ../br_$f; done exit -# and also disable these msc warnings: -# pragma warning(disable : 4100) -# pragma warning(disable : 4127) -# pragma warning(disable : 4295) -# pragma warning(disable : 4389) -# pragma warning(disable : 4701) +# then put these to "port.h" + +/* disable some warnings /TR */ +#pragma warning(disable : 4100) +#pragma warning(disable : 4127) +#pragma warning(disable : 4189) +#pragma warning(disable : 4295) +#pragma warning(disable : 4389) +#pragma warning(disable : 4701) diff --git a/C/brotli/br_backward_references_hq.c b/C/brotli/br_backward_references_hq.c index ccf290a0..481008e4 100644 --- a/C/brotli/br_backward_references_hq.c +++ b/C/brotli/br_backward_references_hq.c @@ -330,7 +330,7 @@ static size_t ComputeMinimumCopyLength(const float start_cost, REQUIRES: nodes[0..pos] satisfies that "ZopfliNode array invariant". */ static uint32_t ComputeDistanceShortcut(const size_t block_start, const size_t pos, - const size_t max_backward, + const size_t max_backward_limit, const size_t gap, const ZopfliNode* nodes) { const size_t clen = ZopfliNodeCopyLength(&nodes[pos]); @@ -338,13 +338,13 @@ static uint32_t ComputeDistanceShortcut(const size_t block_start, const size_t dist = ZopfliNodeCopyDistance(&nodes[pos]); /* Since |block_start + pos| is the end position of the command, the copy part starts from |block_start + pos - clen|. Distances that are greater than - this or greater than |max_backward| are static dictionary references, and - do not update the last distances. Also distance code 0 (last distance) - does not update the last distances. */ + this or greater than |max_backward_limit| + |gap| are static dictionary + references, and do not update the last distances. + Also distance code 0 (last distance) does not update the last distances. */ if (pos == 0) { return 0; } else if (dist + clen <= block_start + pos + gap && - dist <= max_backward + gap && + dist <= max_backward_limit + gap && ZopfliNodeDistanceCode(&nodes[pos]) > 0) { return (uint32_t)pos; } else { @@ -454,9 +454,11 @@ static size_t UpdateNodes( break; } if (BROTLI_PREDICT_FALSE(backward > max_distance + gap)) { + /* Word dictionary -> ignore. */ continue; } if (backward <= max_distance) { + /* Regular backward reference. */ if (prev_ix >= cur_ix) { continue; } @@ -564,14 +566,10 @@ static size_t ComputeShortestPathFromNodes(size_t num_bytes, /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ void BrotliZopfliCreateCommands(const size_t num_bytes, - const size_t block_start, - const size_t max_backward_limit, - const ZopfliNode* nodes, - int* dist_cache, - size_t* last_insert_len, - const BrotliEncoderParams* params, - Command* commands, - size_t* num_literals) { + const size_t block_start, const ZopfliNode* nodes, int* dist_cache, + size_t* last_insert_len, const BrotliEncoderParams* params, + Command* commands, size_t* num_literals) { + const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); size_t pos = 0; uint32_t offset = nodes[0].u.next; size_t i; @@ -610,18 +608,12 @@ void BrotliZopfliCreateCommands(const size_t num_bytes, *last_insert_len += num_bytes - pos; } -static size_t ZopfliIterate(size_t num_bytes, - size_t position, - const uint8_t* ringbuffer, - size_t ringbuffer_mask, - const BrotliEncoderParams* params, - const size_t max_backward_limit, - const size_t gap, - const int* dist_cache, - const ZopfliCostModel* model, - const uint32_t* num_matches, - const BackwardMatch* matches, - ZopfliNode* nodes) { +static size_t ZopfliIterate(size_t num_bytes, size_t position, + const uint8_t* ringbuffer, size_t ringbuffer_mask, + const BrotliEncoderParams* params, const size_t gap, const int* dist_cache, + const ZopfliCostModel* model, const uint32_t* num_matches, + const BackwardMatch* matches, ZopfliNode* nodes) { + const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); const size_t max_zopfli_len = MaxZopfliLen(params); StartPosQueue queue; size_t cur_match_pos = 0; @@ -645,8 +637,8 @@ static size_t ZopfliIterate(size_t num_bytes, while (skip) { i++; if (i + 3 >= num_bytes) break; - EvaluateNode(position, i, max_backward_limit, gap, dist_cache, model, - &queue, nodes); + EvaluateNode(position, i, max_backward_limit, gap, + dist_cache, model, &queue, nodes); cur_match_pos += num_matches[i]; skip--; } @@ -656,11 +648,11 @@ static size_t ZopfliIterate(size_t num_bytes, } /* REQUIRES: nodes != NULL and len(nodes) >= num_bytes + 1 */ -size_t BrotliZopfliComputeShortestPath(MemoryManager* m, - size_t num_bytes, size_t position, const uint8_t* ringbuffer, - size_t ringbuffer_mask, const BrotliEncoderParams* params, - const size_t max_backward_limit, const int* dist_cache, HasherHandle hasher, - ZopfliNode* nodes) { +size_t BrotliZopfliComputeShortestPath(MemoryManager* m, size_t num_bytes, + size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, + const BrotliEncoderParams* params, + const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes) { + const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); const size_t max_zopfli_len = MaxZopfliLen(params); ZopfliCostModel model; StartPosQueue queue; @@ -681,9 +673,11 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, const size_t pos = position + i; const size_t max_distance = BROTLI_MIN(size_t, pos, max_backward_limit); size_t skip; - size_t num_matches = FindAllMatchesH10(hasher, ¶ms->dictionary, - ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance, gap, - params, &matches[lz_matches_offset]); + size_t num_matches; + num_matches = FindAllMatchesH10(hasher, + ¶ms->dictionary, + ringbuffer, ringbuffer_mask, pos, num_bytes - i, max_distance, + gap, params, &matches[lz_matches_offset]); if (num_matches > 0 && BackwardMatchLength(&matches[num_matches - 1]) > max_zopfli_len) { matches[0] = matches[num_matches - 1]; @@ -704,8 +698,8 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, while (skip) { i++; if (i + HashTypeLengthH10() - 1 >= num_bytes) break; - EvaluateNode(position, i, max_backward_limit, gap, dist_cache, &model, - &queue, nodes); + EvaluateNode(position, i, max_backward_limit, gap, + dist_cache, &model, &queue, nodes); skip--; } } @@ -714,28 +708,27 @@ size_t BrotliZopfliComputeShortestPath(MemoryManager* m, return ComputeShortestPathFromNodes(num_bytes, nodes); } -void BrotliCreateZopfliBackwardReferences(MemoryManager* m, - size_t num_bytes, size_t position, const uint8_t* ringbuffer, - size_t ringbuffer_mask, const BrotliEncoderParams* params, +void BrotliCreateZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, + size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, + const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache, size_t* last_insert_len, Command* commands, size_t* num_commands, size_t* num_literals) { - const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); ZopfliNode* nodes; nodes = BROTLI_ALLOC(m, ZopfliNode, num_bytes + 1); if (BROTLI_IS_OOM(m)) return; BrotliInitZopfliNodes(nodes, num_bytes + 1); - *num_commands += BrotliZopfliComputeShortestPath(m, - num_bytes, position, ringbuffer, ringbuffer_mask, - params, max_backward_limit, dist_cache, hasher, nodes); + *num_commands += BrotliZopfliComputeShortestPath(m, num_bytes, + position, ringbuffer, ringbuffer_mask, params, + dist_cache, hasher, nodes); if (BROTLI_IS_OOM(m)) return; - BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, nodes, - dist_cache, last_insert_len, params, commands, num_literals); + BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache, + last_insert_len, params, commands, num_literals); BROTLI_FREE(m, nodes); } -void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, - size_t num_bytes, size_t position, const uint8_t* ringbuffer, - size_t ringbuffer_mask, const BrotliEncoderParams* params, +void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, size_t num_bytes, + size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, + const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache, size_t* last_insert_len, Command* commands, size_t* num_commands, size_t* num_literals) { const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); @@ -767,8 +760,10 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, cur_match_pos + MAX_NUM_MATCHES_H10 + shadow_matches); if (BROTLI_IS_OOM(m)) return; num_found_matches = FindAllMatchesH10(hasher, - ¶ms->dictionary, ringbuffer, ringbuffer_mask, pos, max_length, - max_distance, gap, params, &matches[cur_match_pos + shadow_matches]); + ¶ms->dictionary, + ringbuffer, ringbuffer_mask, pos, max_length, + max_distance, gap, params, + &matches[cur_match_pos + shadow_matches]); cur_match_end = cur_match_pos + num_found_matches; for (j = cur_match_pos; j + 1 < cur_match_end; ++j) { BROTLI_DCHECK(BackwardMatchLength(&matches[j]) <= @@ -814,10 +809,10 @@ void BrotliCreateHqZopfliBackwardReferences(MemoryManager* m, *last_insert_len = orig_last_insert_len; memcpy(dist_cache, orig_dist_cache, 4 * sizeof(dist_cache[0])); *num_commands += ZopfliIterate(num_bytes, position, ringbuffer, - ringbuffer_mask, params, max_backward_limit, gap, dist_cache, - &model, num_matches, matches, nodes); - BrotliZopfliCreateCommands(num_bytes, position, max_backward_limit, - nodes, dist_cache, last_insert_len, params, commands, num_literals); + ringbuffer_mask, params, gap, dist_cache, &model, num_matches, matches, + nodes); + BrotliZopfliCreateCommands(num_bytes, position, nodes, dist_cache, + last_insert_len, params, commands, num_literals); } CleanupZopfliCostModel(m, &model); BROTLI_FREE(m, nodes); diff --git a/C/brotli/br_decode.c b/C/brotli/br_decode.c index 5249144d..11604868 100644 --- a/C/brotli/br_decode.c +++ b/C/brotli/br_decode.c @@ -6,10 +6,6 @@ #include "decode.h" -#if defined(__ARM_NEON__) -#include -#endif - #include /* free, malloc */ #include /* memcpy, memset */ @@ -24,6 +20,10 @@ #include "./dec/prefix.h" #include "./dec/state.h" +#if defined(BROTLI_TARGET_NEON) +#include +#endif + #if defined(__cplusplus) || defined(c_plusplus) extern "C" { #endif @@ -167,7 +167,7 @@ static BrotliDecoderErrorCode DecodeWindowBits(BrotliDecoderState* s, } static BROTLI_INLINE void memmove16(uint8_t* dst, uint8_t* src) { -#if defined(__ARM_NEON__) +#if defined(BROTLI_TARGET_NEON) vst1q_u8(dst, vld1q_u8(src)); #else uint32_t buffer[4]; @@ -347,15 +347,17 @@ static BrotliDecoderErrorCode BROTLI_NOINLINE DecodeMetaBlockLength( static BROTLI_INLINE uint32_t DecodeSymbol(uint32_t bits, const HuffmanCode* table, BrotliBitReader* br) { - table += bits & HUFFMAN_TABLE_MASK; - if (table->bits > HUFFMAN_TABLE_BITS) { - uint32_t nbits = table->bits - HUFFMAN_TABLE_BITS; + BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table); + BROTLI_HC_ADJUST_TABLE_INDEX(table, bits & HUFFMAN_TABLE_MASK); + if (BROTLI_HC_FAST_LOAD_BITS(table) > HUFFMAN_TABLE_BITS) { + uint32_t nbits = BROTLI_HC_FAST_LOAD_BITS(table) - HUFFMAN_TABLE_BITS; BrotliDropBits(br, HUFFMAN_TABLE_BITS); - table += table->value; - table += (bits >> HUFFMAN_TABLE_BITS) & BitMask(nbits); + BROTLI_HC_ADJUST_TABLE_INDEX(table, + BROTLI_HC_FAST_LOAD_VALUE(table) + + ((bits >> HUFFMAN_TABLE_BITS) & BitMask(nbits))); } - BrotliDropBits(br, table->bits); - return table->value; + BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(table)); + return BROTLI_HC_FAST_LOAD_VALUE(table); } /* Reads and decodes the next Huffman code from bit-stream. @@ -371,19 +373,20 @@ static BROTLI_NOINLINE BROTLI_BOOL SafeDecodeSymbol( const HuffmanCode* table, BrotliBitReader* br, uint32_t* result) { uint32_t val; uint32_t available_bits = BrotliGetAvailableBits(br); + BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table); if (available_bits == 0) { - if (table->bits == 0) { - *result = table->value; + if (BROTLI_HC_FAST_LOAD_BITS(table) == 0) { + *result = BROTLI_HC_FAST_LOAD_VALUE(table); return BROTLI_TRUE; } return BROTLI_FALSE; /* No valid bits at all. */ } val = (uint32_t)BrotliGetBitsUnmasked(br); - table += val & HUFFMAN_TABLE_MASK; - if (table->bits <= HUFFMAN_TABLE_BITS) { - if (table->bits <= available_bits) { - BrotliDropBits(br, table->bits); - *result = table->value; + BROTLI_HC_ADJUST_TABLE_INDEX(table, val & HUFFMAN_TABLE_MASK); + if (BROTLI_HC_FAST_LOAD_BITS(table) <= HUFFMAN_TABLE_BITS) { + if (BROTLI_HC_FAST_LOAD_BITS(table) <= available_bits) { + BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(table)); + *result = BROTLI_HC_FAST_LOAD_VALUE(table); return BROTLI_TRUE; } else { return BROTLI_FALSE; /* Not enough bits for the first level. */ @@ -394,15 +397,15 @@ static BROTLI_NOINLINE BROTLI_BOOL SafeDecodeSymbol( } /* Speculatively drop HUFFMAN_TABLE_BITS. */ - val = (val & BitMask(table->bits)) >> HUFFMAN_TABLE_BITS; + val = (val & BitMask(BROTLI_HC_FAST_LOAD_BITS(table))) >> HUFFMAN_TABLE_BITS; available_bits -= HUFFMAN_TABLE_BITS; - table += table->value + val; - if (available_bits < table->bits) { + BROTLI_HC_ADJUST_TABLE_INDEX(table, BROTLI_HC_FAST_LOAD_VALUE(table) + val); + if (available_bits < BROTLI_HC_FAST_LOAD_BITS(table)) { return BROTLI_FALSE; /* Not enough bits for the second level. */ } - BrotliDropBits(br, HUFFMAN_TABLE_BITS + table->bits); - *result = table->value; + BrotliDropBits(br, HUFFMAN_TABLE_BITS + BROTLI_HC_FAST_LOAD_BITS(table)); + *result = BROTLI_HC_FAST_LOAD_VALUE(table); return BROTLI_TRUE; } @@ -425,9 +428,10 @@ static BROTLI_INLINE void PreloadSymbol(int safe, if (safe) { return; } - table += BrotliGetBits(br, HUFFMAN_TABLE_BITS); - *bits = table->bits; - *value = table->value; + BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table); + BROTLI_HC_ADJUST_TABLE_INDEX(table, BrotliGetBits(br, HUFFMAN_TABLE_BITS)); + *bits = BROTLI_HC_FAST_LOAD_BITS(table); + *value = BROTLI_HC_FAST_LOAD_VALUE(table); } /* Decodes the next Huffman code using data prepared by PreloadSymbol. @@ -441,10 +445,11 @@ static BROTLI_INLINE uint32_t ReadPreloadedSymbol(const HuffmanCode* table, uint32_t val = BrotliGet16BitsUnmasked(br); const HuffmanCode* ext = table + (val & HUFFMAN_TABLE_MASK) + *value; uint32_t mask = BitMask((*bits - HUFFMAN_TABLE_BITS)); + BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(ext); BrotliDropBits(br, HUFFMAN_TABLE_BITS); - ext += (val >> HUFFMAN_TABLE_BITS) & mask; - BrotliDropBits(br, ext->bits); - result = ext->value; + BROTLI_HC_ADJUST_TABLE_INDEX(ext, (val >> HUFFMAN_TABLE_BITS) & mask); + BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(ext)); + result = BROTLI_HC_FAST_LOAD_VALUE(ext); } else { BrotliDropBits(br, *bits); } @@ -597,6 +602,7 @@ static BrotliDecoderErrorCode ReadSymbolCodeLengths( while (symbol < alphabet_size && space > 0) { const HuffmanCode* p = s->table; uint32_t code_len; + BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(p); if (!BrotliCheckInputAmount(br, BROTLI_SHORT_FILL_BIT_WINDOW_READ)) { s->symbol = symbol; s->repeat = repeat; @@ -606,10 +612,10 @@ static BrotliDecoderErrorCode ReadSymbolCodeLengths( return BROTLI_DECODER_NEEDS_MORE_INPUT; } BrotliFillBitWindow16(br); - p += BrotliGetBitsUnmasked(br) & - BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH); - BrotliDropBits(br, p->bits); /* Use 1..5 bits. */ - code_len = p->value; /* code_len == 0..17 */ + BROTLI_HC_ADJUST_TABLE_INDEX(p, BrotliGetBitsUnmasked(br) & + BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH)); + BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(p)); /* Use 1..5 bits. */ + code_len = BROTLI_HC_FAST_LOAD_VALUE(p); /* code_len == 0..17 */ if (code_len < BROTLI_REPEAT_PREVIOUS_CODE_LENGTH) { ProcessSingleCodeLength(code_len, &symbol, &repeat, &space, &prev_code_len, symbol_lists, code_length_histo, next_symbol); @@ -637,31 +643,34 @@ static BrotliDecoderErrorCode SafeReadSymbolCodeLengths( uint32_t code_len; uint32_t available_bits; uint32_t bits = 0; + BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(p); if (get_byte && !BrotliPullByte(br)) return BROTLI_DECODER_NEEDS_MORE_INPUT; get_byte = BROTLI_FALSE; available_bits = BrotliGetAvailableBits(br); if (available_bits != 0) { bits = (uint32_t)BrotliGetBitsUnmasked(br); } - p += bits & BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH); - if (p->bits > available_bits) { + BROTLI_HC_ADJUST_TABLE_INDEX(p, + bits & BitMask(BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH)); + if (BROTLI_HC_FAST_LOAD_BITS(p) > available_bits) { get_byte = BROTLI_TRUE; continue; } - code_len = p->value; /* code_len == 0..17 */ + code_len = BROTLI_HC_FAST_LOAD_VALUE(p); /* code_len == 0..17 */ if (code_len < BROTLI_REPEAT_PREVIOUS_CODE_LENGTH) { - BrotliDropBits(br, p->bits); + BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(p)); ProcessSingleCodeLength(code_len, &s->symbol, &s->repeat, &s->space, &s->prev_code_len, s->symbol_lists, s->code_length_histo, s->next_symbol); } else { /* code_len == 16..17, extra_bits == 2..3 */ uint32_t extra_bits = code_len - 14U; - uint32_t repeat_delta = (bits >> p->bits) & BitMask(extra_bits); - if (available_bits < p->bits + extra_bits) { + uint32_t repeat_delta = (bits >> BROTLI_HC_FAST_LOAD_BITS(p)) & + BitMask(extra_bits); + if (available_bits < BROTLI_HC_FAST_LOAD_BITS(p) + extra_bits) { get_byte = BROTLI_TRUE; continue; } - BrotliDropBits(br, p->bits + extra_bits); + BrotliDropBits(br, BROTLI_HC_FAST_LOAD_BITS(p) + extra_bits); ProcessRepeatedCodeLength(code_len, repeat_delta, alphabet_size, &s->symbol, &s->repeat, &s->space, &s->prev_code_len, &s->repeat_code_len, s->symbol_lists, s->code_length_histo, diff --git a/C/brotli/br_encode.c b/C/brotli/br_encode.c index f4cf3273..9c0573f1 100644 --- a/C/brotli/br_encode.c +++ b/C/brotli/br_encode.c @@ -496,6 +496,8 @@ static void DecideOverLiteralContextModeling(const uint8_t* input, static BROTLI_BOOL ShouldCompress( const uint8_t* data, const size_t mask, const uint64_t last_flush_pos, const size_t bytes, const size_t num_literals, const size_t num_commands) { + /* TODO: find more precise minimal block overhead. */ + if (bytes <= 2) return BROTLI_FALSE; if (num_commands < (bytes >> 8) + 2) { if (num_literals > 0.99 * (double)bytes) { uint32_t literal_histo[256] = { 0 }; @@ -674,12 +676,14 @@ static BROTLI_BOOL EnsureInitialized(BrotliEncoderState* s) { if (BROTLI_IS_OOM(&s->memory_manager_)) return BROTLI_FALSE; if (s->is_initialized_) return BROTLI_TRUE; + s->last_bytes_bits_ = 0; + s->last_bytes_ = 0; + s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX; + SanitizeParams(&s->params); s->params.lgblock = ComputeLgBlock(&s->params); ChooseDistanceParams(&s->params); - s->remaining_metadata_bytes_ = BROTLI_UINT32_MAX; - RingBufferSetup(&s->params, &s->ringbuffer_); /* Initialize last byte with stream header. */ @@ -1029,23 +1033,20 @@ static BROTLI_BOOL EncodeData( if (s->params.quality == ZOPFLIFICATION_QUALITY) { BROTLI_DCHECK(s->params.hasher.type == 10); - BrotliCreateZopfliBackwardReferences(m, - bytes, wrapped_last_processed_pos, + BrotliCreateZopfliBackwardReferences(m, bytes, wrapped_last_processed_pos, data, mask, &s->params, s->hasher_, s->dist_cache_, &s->last_insert_len_, &s->commands_[s->num_commands_], &s->num_commands_, &s->num_literals_); if (BROTLI_IS_OOM(m)) return BROTLI_FALSE; } else if (s->params.quality == HQ_ZOPFLIFICATION_QUALITY) { BROTLI_DCHECK(s->params.hasher.type == 10); - BrotliCreateHqZopfliBackwardReferences(m, - bytes, wrapped_last_processed_pos, + BrotliCreateHqZopfliBackwardReferences(m, bytes, wrapped_last_processed_pos, data, mask, &s->params, s->hasher_, s->dist_cache_, &s->last_insert_len_, &s->commands_[s->num_commands_], &s->num_commands_, &s->num_literals_); if (BROTLI_IS_OOM(m)) return BROTLI_FALSE; } else { - BrotliCreateBackwardReferences( - bytes, wrapped_last_processed_pos, + BrotliCreateBackwardReferences(bytes, wrapped_last_processed_pos, data, mask, &s->params, s->hasher_, s->dist_cache_, &s->last_insert_len_, &s->commands_[s->num_commands_], &s->num_commands_, &s->num_literals_); @@ -1166,7 +1167,6 @@ static BROTLI_BOOL BrotliCompressBufferQuality10( MemoryManager* m = &memory_manager; const size_t mask = BROTLI_SIZE_MAX >> 1; - const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(lgwin); int dist_cache[4] = { 4, 11, 15, 16 }; int saved_dist_cache[4] = { 4, 11, 15, 16 }; BROTLI_BOOL ok = BROTLI_TRUE; @@ -1176,8 +1176,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10( uint8_t last_bytes_bits; HasherHandle hasher = NULL; - const size_t hasher_eff_size = - BROTLI_MIN(size_t, input_size, max_backward_limit + BROTLI_WINDOW_GAP); + const size_t hasher_eff_size = BROTLI_MIN(size_t, + input_size, BROTLI_MAX_BACKWARD_LIMIT(lgwin) + BROTLI_WINDOW_GAP); BrotliEncoderParams params; @@ -1238,9 +1238,9 @@ static BROTLI_BOOL BrotliCompressBufferQuality10( BrotliInitZopfliNodes(nodes, block_size + 1); StitchToPreviousBlockH10(hasher, block_size, block_start, input_buffer, mask); - path_size = BrotliZopfliComputeShortestPath(m, - block_size, block_start, input_buffer, mask, ¶ms, - max_backward_limit, dist_cache, hasher, nodes); + path_size = BrotliZopfliComputeShortestPath(m, block_size, block_start, + input_buffer, mask, ¶ms, dist_cache, hasher, + nodes); if (BROTLI_IS_OOM(m)) goto oom; /* We allocate a command buffer in the first iteration of this loop that will be likely big enough for the whole metablock, so that for most @@ -1262,10 +1262,8 @@ static BROTLI_BOOL BrotliCompressBufferQuality10( } commands = new_commands; } - BrotliZopfliCreateCommands(block_size, block_start, max_backward_limit, - &nodes[0], dist_cache, &last_insert_len, - ¶ms, &commands[num_commands], - &num_literals); + BrotliZopfliCreateCommands(block_size, block_start, &nodes[0], dist_cache, + &last_insert_len, ¶ms, &commands[num_commands], &num_literals); num_commands += path_size; block_start += block_size; metablock_size += block_size; diff --git a/C/brotli/br_huffman.c b/C/brotli/br_huffman.c index 52c109b6..5a9f8fec 100644 --- a/C/brotli/br_huffman.c +++ b/C/brotli/br_huffman.c @@ -142,8 +142,7 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table, /* Special case: all symbols but one have 0 code length. */ if (offset[0] == 0) { - code.bits = 0; - code.value = (uint16_t)sorted[0]; + code = ConstructHuffmanCode(0, (uint16_t)sorted[0]); for (key = 0; key < (brotli_reg_t)table_size; ++key) { table[key] = code; } @@ -157,9 +156,8 @@ void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* table, bits = 1; step = 2; do { - code.bits = (uint8_t)bits; for (bits_count = count[bits]; bits_count != 0; --bits_count) { - code.value = (uint16_t)sorted[symbol++]; + code = ConstructHuffmanCode((uint8_t)bits, (uint16_t)sorted[symbol++]); ReplicateValue(&table[BrotliReverseBits(key)], step, table_size, code); key += key_step; } @@ -211,11 +209,10 @@ uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table, bits = 1; step = 2; do { - code.bits = (uint8_t)bits; symbol = bits - (BROTLI_HUFFMAN_MAX_CODE_LENGTH + 1); for (bits_count = count[bits]; bits_count != 0; --bits_count) { symbol = symbol_lists[symbol]; - code.value = (uint16_t)symbol; + code = ConstructHuffmanCode((uint8_t)bits, (uint16_t)symbol); ReplicateValue(&table[BrotliReverseBits(key)], step, table_size, code); key += key_step; } @@ -244,14 +241,13 @@ uint32_t BrotliBuildHuffmanTable(HuffmanCode* root_table, total_size += table_size; sub_key = BrotliReverseBits(key); key += key_step; - root_table[sub_key].bits = (uint8_t)(table_bits + root_bits); - root_table[sub_key].value = - (uint16_t)(((size_t)(table - root_table)) - sub_key); + root_table[sub_key] = ConstructHuffmanCode( + (uint8_t)(table_bits + root_bits), + (uint16_t)(((size_t)(table - root_table)) - sub_key)); sub_key = 0; } - code.bits = (uint8_t)(len - root_bits); symbol = symbol_lists[symbol]; - code.value = (uint16_t)symbol; + code = ConstructHuffmanCode((uint8_t)(len - root_bits), (uint16_t)symbol); ReplicateValue( &table[BrotliReverseBits(sub_key)], step, table_size, code); sub_key += sub_key_step; @@ -270,35 +266,28 @@ uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table, const uint32_t goal_size = 1U << root_bits; switch (num_symbols) { case 0: - table[0].bits = 0; - table[0].value = val[0]; + table[0] = ConstructHuffmanCode(0, val[0]); break; case 1: - table[0].bits = 1; - table[1].bits = 1; if (val[1] > val[0]) { - table[0].value = val[0]; - table[1].value = val[1]; + table[0] = ConstructHuffmanCode(1, val[0]); + table[1] = ConstructHuffmanCode(1, val[1]); } else { - table[0].value = val[1]; - table[1].value = val[0]; + table[0] = ConstructHuffmanCode(1, val[1]); + table[1] = ConstructHuffmanCode(1, val[0]); } table_size = 2; break; case 2: - table[0].bits = 1; - table[0].value = val[0]; - table[2].bits = 1; - table[2].value = val[0]; + table[0] = ConstructHuffmanCode(1, val[0]); + table[2] = ConstructHuffmanCode(1, val[0]); if (val[2] > val[1]) { - table[1].value = val[1]; - table[3].value = val[2]; + table[1] = ConstructHuffmanCode(2, val[1]); + table[3] = ConstructHuffmanCode(2, val[2]); } else { - table[1].value = val[2]; - table[3].value = val[1]; + table[1] = ConstructHuffmanCode(2, val[2]); + table[3] = ConstructHuffmanCode(2, val[1]); } - table[1].bits = 2; - table[3].bits = 2; table_size = 4; break; case 3: { @@ -312,33 +301,27 @@ uint32_t BrotliBuildSimpleHuffmanTable(HuffmanCode* table, } } } - for (i = 0; i < 4; ++i) { - table[i].bits = 2; - } - table[0].value = val[0]; - table[2].value = val[1]; - table[1].value = val[2]; - table[3].value = val[3]; + table[0] = ConstructHuffmanCode(2, val[0]); + table[2] = ConstructHuffmanCode(2, val[1]); + table[1] = ConstructHuffmanCode(2, val[2]); + table[3] = ConstructHuffmanCode(2, val[3]); table_size = 4; break; } case 4: { - int i; if (val[3] < val[2]) { uint16_t t = val[3]; val[3] = val[2]; val[2] = t; } - for (i = 0; i < 7; ++i) { - table[i].value = val[0]; - table[i].bits = (uint8_t)(1 + (i & 1)); - } - table[1].value = val[1]; - table[3].value = val[2]; - table[5].value = val[1]; - table[7].value = val[3]; - table[3].bits = 3; - table[7].bits = 3; + table[0] = ConstructHuffmanCode(1, val[0]); + table[1] = ConstructHuffmanCode(2, val[1]); + table[2] = ConstructHuffmanCode(1, val[0]); + table[3] = ConstructHuffmanCode(3, val[2]); + table[4] = ConstructHuffmanCode(1, val[0]); + table[5] = ConstructHuffmanCode(2, val[1]); + table[6] = ConstructHuffmanCode(1, val[0]); + table[7] = ConstructHuffmanCode(3, val[3]); table_size = 8; break; } diff --git a/C/brotli/br_transform.c b/C/brotli/br_transform.c index eabd367a..cb0d74ac 100644 --- a/C/brotli/br_transform.c +++ b/C/brotli/br_transform.c @@ -191,11 +191,11 @@ static int ToUpperCase(uint8_t* p) { } int BrotliTransformDictionaryWord(uint8_t* dst, const uint8_t* word, int len, - const BrotliTransforms* transforms, int transfom_idx) { + const BrotliTransforms* transforms, int transform_idx) { int idx = 0; - const uint8_t* prefix = BROTLI_TRANSFORM_PREFIX(transforms, transfom_idx); - uint8_t type = BROTLI_TRANSFORM_TYPE(transforms, transfom_idx); - const uint8_t* suffix = BROTLI_TRANSFORM_SUFFIX(transforms, transfom_idx); + const uint8_t* prefix = BROTLI_TRANSFORM_PREFIX(transforms, transform_idx); + uint8_t type = BROTLI_TRANSFORM_TYPE(transforms, transform_idx); + const uint8_t* suffix = BROTLI_TRANSFORM_SUFFIX(transforms, transform_idx); { int prefix_len = *prefix++; while (prefix_len--) { dst[idx++] = *prefix++; } diff --git a/C/brotli/common/platform.h b/C/brotli/common/platform.h index 252889b1..5fb3c8c9 100644 --- a/C/brotli/common/platform.h +++ b/C/brotli/common/platform.h @@ -71,7 +71,7 @@ OR: */ #if BROTLI_GNUC_HAS_BUILTIN(__builtin_expect, 3, 0, 0) || \ BROTLI_INTEL_VERSION_CHECK(16, 0, 0) || \ - BROTLI_SUNPRO_VERSION_CHECK(5, 12, 0) || \ + BROTLI_SUNPRO_VERSION_CHECK(5, 15, 0) || \ BROTLI_ARM_VERSION_CHECK(4, 1, 0) || \ BROTLI_IBM_VERSION_CHECK(10, 1, 0) || \ BROTLI_TI_VERSION_CHECK(7, 3, 0) || \ @@ -180,6 +180,12 @@ OR: #define BROTLI_UNUSED_FUNCTION static BROTLI_INLINE #endif +#if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) +#define BROTLI_ALIGNED(N) __attribute__((aligned(N))) +#else +#define BROTLI_ALIGNED(N) +#endif + #if (defined(__ARM_ARCH) && (__ARM_ARCH == 7)) || \ (defined(M_ARM) && (M_ARM == 7)) #define BROTLI_TARGET_ARMV7 @@ -197,6 +203,10 @@ OR: #endif /* ARMv8 */ +#if defined(__ARM_NEON__) || defined(__ARM_NEON) +#define BROTLI_TARGET_NEON +#endif + #if defined(__i386) || defined(_M_IX86) #define BROTLI_TARGET_X86 #endif @@ -343,7 +353,7 @@ static BROTLI_INLINE void BrotliUnalignedWrite64(void* p, uint64_t v) { /* If __attribute__(aligned) is available, use that. Otherwise, memcpy. */ #if BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0) -typedef __attribute__((aligned(1))) uint64_t brotli_unaligned_uint64_t; +typedef BROTLI_ALIGNED(1) uint64_t brotli_unaligned_uint64_t; static BROTLI_INLINE uint64_t BrotliUnalignedRead64(const void* p) { return (uint64_t) ((brotli_unaligned_uint64_t*) p)[0]; diff --git a/C/brotli/common/version.h b/C/brotli/common/version.h index 1ba0806f..0d0d0c79 100644 --- a/C/brotli/common/version.h +++ b/C/brotli/common/version.h @@ -14,13 +14,13 @@ BrotliEncoderVersion methods. */ /* Semantic version, calculated as (MAJOR << 24) | (MINOR << 12) | PATCH */ -#define BROTLI_VERSION 0x1000006 +#define BROTLI_VERSION 0x1000007 /* This macro is used by build system to produce Libtool-friendly soname. See https://www.gnu.org/software/libtool/manual/html_node/Libtool-versioning.html */ /* ABI version, calculated as (CURRENT << 24) | (REVISION << 12) | AGE */ -#define BROTLI_ABI_VERSION 0x1006000 +#define BROTLI_ABI_VERSION 0x1007000 #endif /* BROTLI_COMMON_VERSION_H_ */ diff --git a/C/brotli/dec/huffman.h b/C/brotli/dec/huffman.h index 5dba8128..8f71db11 100644 --- a/C/brotli/dec/huffman.h +++ b/C/brotli/dec/huffman.h @@ -33,11 +33,66 @@ static const uint16_t kMaxHuffmanTableSize[] = { #define BROTLI_HUFFMAN_MAX_CODE_LENGTH_CODE_LENGTH 5 +#if ((defined(BROTLI_TARGET_ARMV7) || defined(BROTLI_TARGET_ARMV8_32)) && \ + BROTLI_GNUC_HAS_ATTRIBUTE(aligned, 2, 7, 0)) +#define BROTLI_HUFFMAN_CODE_FAST_LOAD +#endif + +#if !defined(BROTLI_HUFFMAN_CODE_FAST_LOAD) +/* Do not create this struct directly - use the ConstructHuffmanCode + * constructor below! */ typedef struct { uint8_t bits; /* number of bits used for this symbol */ uint16_t value; /* symbol value or table offset */ } HuffmanCode; +static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits, + const uint16_t value) { + HuffmanCode h; + h.bits = bits; + h.value = value; + return h; +} + +/* Please use the following macros to optimize HuffmanCode accesses in hot + * paths. + * + * For example, assuming |table| contains a HuffmanCode pointer: + * + * BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(table); + * BROTLI_HC_ADJUST_TABLE_INDEX(table, index_into_table); + * *bits = BROTLI_HC_GET_BITS(table); + * *value = BROTLI_HC_GET_VALUE(table); + * BROTLI_HC_ADJUST_TABLE_INDEX(table, offset); + * *bits2 = BROTLI_HC_GET_BITS(table); + * *value2 = BROTLI_HC_GET_VALUE(table); + * + */ + +#define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) +#define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V) + +/* These must be given a HuffmanCode pointer! */ +#define BROTLI_HC_FAST_LOAD_BITS(H) (H->bits) +#define BROTLI_HC_FAST_LOAD_VALUE(H) (H->value) + +#else /* BROTLI_HUFFMAN_CODE_FAST_LOAD */ + +typedef BROTLI_ALIGNED(4) uint32_t HuffmanCode; + +static BROTLI_INLINE HuffmanCode ConstructHuffmanCode(const uint8_t bits, + const uint16_t value) { + return ((value & 0xFFFF) << 16) | (bits & 0xFF); +} + +#define BROTLI_HC_MARK_TABLE_FOR_FAST_LOAD(H) uint32_t __fastload_##H = (*H) +#define BROTLI_HC_ADJUST_TABLE_INDEX(H, V) H += (V); __fastload_##H = (*H) + +/* These must be given a HuffmanCode pointer! */ +#define BROTLI_HC_FAST_LOAD_BITS(H) ((__fastload_##H) & 0xFF) +#define BROTLI_HC_FAST_LOAD_VALUE(H) ((__fastload_##H) >> 16) +#endif /* BROTLI_HUFFMAN_CODE_FAST_LOAD */ + /* Builds Huffman lookup table assuming code lengths are in symbol order. */ BROTLI_INTERNAL void BrotliBuildCodeLengthsHuffmanTable(HuffmanCode* root_table, const uint8_t* const code_lengths, uint16_t* count); diff --git a/C/brotli/enc/backward_references_hq.h b/C/brotli/enc/backward_references_hq.h index 6aa87fe5..c70881c6 100644 --- a/C/brotli/enc/backward_references_hq.h +++ b/C/brotli/enc/backward_references_hq.h @@ -74,15 +74,14 @@ BROTLI_INTERNAL void BrotliInitZopfliNodes(ZopfliNode* array, size_t length); (1) nodes[i].copy_length() >= 2 (2) nodes[i].command_length() <= i and (3) nodes[i - nodes[i].command_length()].cost < kInfinity */ -BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath(MemoryManager* m, - size_t num_bytes, size_t position, const uint8_t* ringbuffer, - size_t ringbuffer_mask, const BrotliEncoderParams* params, - const size_t max_backward_limit, const int* dist_cache, HasherHandle hasher, - ZopfliNode* nodes); +BROTLI_INTERNAL size_t BrotliZopfliComputeShortestPath( + MemoryManager* m, size_t num_bytes, + size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, + const BrotliEncoderParams* params, + const int* dist_cache, HasherHandle hasher, ZopfliNode* nodes); BROTLI_INTERNAL void BrotliZopfliCreateCommands( - const size_t num_bytes, const size_t block_start, - const size_t max_backward_limit, const ZopfliNode* nodes, + const size_t num_bytes, const size_t block_start, const ZopfliNode* nodes, int* dist_cache, size_t* last_insert_len, const BrotliEncoderParams* params, Command* commands, size_t* num_literals); diff --git a/C/brotli/enc/backward_references_inc.h b/C/brotli/enc/backward_references_inc.h index 38a48d39..c18cdb00 100644 --- a/C/brotli/enc/backward_references_inc.h +++ b/C/brotli/enc/backward_references_inc.h @@ -10,9 +10,9 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)( size_t num_bytes, size_t position, const uint8_t* ringbuffer, size_t ringbuffer_mask, - const BrotliEncoderParams* params, HasherHandle hasher, int* dist_cache, - size_t* last_insert_len, Command* commands, size_t* num_commands, - size_t* num_literals) { + const BrotliEncoderParams* params, + HasherHandle hasher, int* dist_cache, size_t* last_insert_len, + Command* commands, size_t* num_commands, size_t* num_literals) { /* Set maximum distance, see section 9.1. of the spec. */ const size_t max_backward_limit = BROTLI_MAX_BACKWARD_LIMIT(params->lgwin); @@ -42,9 +42,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)( sr.distance = 0; sr.score = kMinScore; FN(FindLongestMatch)(hasher, ¶ms->dictionary, - ringbuffer, ringbuffer_mask, dist_cache, position, - max_length, max_distance, gap, - params->dist.max_distance, &sr); + ringbuffer, ringbuffer_mask, dist_cache, position, max_length, + max_distance, gap, params->dist.max_distance, &sr); if (sr.score > kMinScore) { /* Found a match. Let's look for something even better ahead. */ int delayed_backward_references_in_row = 0; @@ -58,7 +57,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)( sr2.distance = 0; sr2.score = kMinScore; max_distance = BROTLI_MIN(size_t, position + 1, max_backward_limit); - FN(FindLongestMatch)(hasher, ¶ms->dictionary, + FN(FindLongestMatch)(hasher, + ¶ms->dictionary, ringbuffer, ringbuffer_mask, dist_cache, position + 1, max_length, max_distance, gap, params->dist.max_distance, &sr2); if (sr2.score >= sr.score + cost_diff_lazy) { @@ -80,8 +80,8 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)( { /* The first 16 codes are special short-codes, and the minimum offset is 1. */ - size_t distance_code = - ComputeDistanceCode(sr.distance, max_distance + gap, dist_cache); + size_t distance_code = ComputeDistanceCode( + sr.distance, max_distance + gap, dist_cache); if ((sr.distance <= (max_distance + gap)) && distance_code > 0) { dist_cache[3] = dist_cache[2]; dist_cache[2] = dist_cache[1]; diff --git a/C/brotli/enc/hash.h b/C/brotli/enc/hash.h index 666afa74..7bf22c63 100644 --- a/C/brotli/enc/hash.h +++ b/C/brotli/enc/hash.h @@ -149,9 +149,9 @@ static BROTLI_INLINE score_t BackwardReferencePenaltyUsingLastDistance( } static BROTLI_INLINE BROTLI_BOOL TestStaticDictionaryItem( - const BrotliEncoderDictionary* dictionary, size_t item, const uint8_t* data, - size_t max_length, size_t max_backward, size_t max_distance, - HasherSearchResult* out) { + const BrotliEncoderDictionary* dictionary, size_t item, + const uint8_t* data, size_t max_length, size_t max_backward, + size_t max_distance, HasherSearchResult* out) { size_t len; size_t word_idx; size_t offset; @@ -208,7 +208,8 @@ static BROTLI_INLINE void SearchInStaticDictionary( self->dict_num_lookups++; if (item != 0) { BROTLI_BOOL item_matches = TestStaticDictionaryItem( - dictionary, item, data, max_length, max_backward, max_distance, out); + dictionary, item, data, + max_length, max_backward, max_distance, out); if (item_matches) { self->dict_num_matches++; } diff --git a/C/brotli/enc/hash_composite_inc.h b/C/brotli/enc/hash_composite_inc.h index f829a97e..b266aa2f 100644 --- a/C/brotli/enc/hash_composite_inc.h +++ b/C/brotli/enc/hash_composite_inc.h @@ -121,13 +121,16 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const BrotliEncoderDictionary* dictionary, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, - const size_t max_length, const size_t max_backward, const size_t gap, - const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) { + const size_t max_length, const size_t max_backward, + const size_t gap, const size_t max_distance, + HasherSearchResult* BROTLI_RESTRICT out) { HashComposite* self = FN(Self)(handle); FN_A(FindLongestMatch)(self->ha, dictionary, data, ring_buffer_mask, - distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out); + distance_cache, cur_ix, max_length, max_backward, gap, + max_distance, out); FN_B(FindLongestMatch)(self->hb, dictionary, data, ring_buffer_mask, - distance_cache, cur_ix, max_length, max_backward, gap, max_distance, out); + distance_cache, cur_ix, max_length, max_backward, gap, + max_distance, out); } #undef HashComposite diff --git a/C/brotli/enc/hash_longest_match64_inc.h b/C/brotli/enc/hash_longest_match64_inc.h index e099edf6..cb953a64 100644 --- a/C/brotli/enc/hash_longest_match64_inc.h +++ b/C/brotli/enc/hash_longest_match64_inc.h @@ -161,8 +161,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const BrotliEncoderDictionary* dictionary, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, - const size_t max_length, const size_t max_backward, const size_t gap, - const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) { + const size_t max_length, const size_t max_backward, + const size_t gap, const size_t max_distance, + HasherSearchResult* BROTLI_RESTRICT out) { HasherCommon* common = GetHasherCommon(handle); HashLongestMatch* self = FN(Self)(handle); uint16_t* num = FN(Num)(self); diff --git a/C/brotli/enc/hash_longest_match_inc.h b/C/brotli/enc/hash_longest_match_inc.h index 951d7a4b..457f5a9e 100644 --- a/C/brotli/enc/hash_longest_match_inc.h +++ b/C/brotli/enc/hash_longest_match_inc.h @@ -154,8 +154,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const BrotliEncoderDictionary* dictionary, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, - const size_t max_length, const size_t max_backward, const size_t gap, - const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) { + const size_t max_length, const size_t max_backward, + const size_t gap, const size_t max_distance, + HasherSearchResult* BROTLI_RESTRICT out) { HasherCommon* common = GetHasherCommon(handle); HashLongestMatch* self = FN(Self)(handle); uint16_t* num = FN(Num)(self); diff --git a/C/brotli/enc/hash_rolling_inc.h b/C/brotli/enc/hash_rolling_inc.h index 4d5d14a0..17f8a408 100644 --- a/C/brotli/enc/hash_rolling_inc.h +++ b/C/brotli/enc/hash_rolling_inc.h @@ -155,8 +155,9 @@ static BROTLI_INLINE void FN(FindLongestMatch)(HasherHandle handle, const BrotliEncoderDictionary* dictionary, const uint8_t* BROTLI_RESTRICT data, const size_t ring_buffer_mask, const int* BROTLI_RESTRICT distance_cache, const size_t cur_ix, - const size_t max_length, const size_t max_backward, const size_t gap, - const size_t max_distance, HasherSearchResult* BROTLI_RESTRICT out) { + const size_t max_length, const size_t max_backward, + const size_t gap, const size_t max_distance, + HasherSearchResult* BROTLI_RESTRICT out) { HashRolling* self = FN(Self)(handle); const size_t cur_ix_masked = cur_ix & ring_buffer_mask; size_t pos = self->next_ix; diff --git a/C/brotli/enc/hash_to_binary_tree_inc.h b/C/brotli/enc/hash_to_binary_tree_inc.h index 48097b17..7fb0356f 100644 --- a/C/brotli/enc/hash_to_binary_tree_inc.h +++ b/C/brotli/enc/hash_to_binary_tree_inc.h @@ -202,8 +202,9 @@ static BROTLI_INLINE BackwardMatch* FN(StoreAndFindMatches)( static BROTLI_INLINE size_t FN(FindAllMatches)(HasherHandle handle, const BrotliEncoderDictionary* dictionary, const uint8_t* data, const size_t ring_buffer_mask, const size_t cur_ix, - const size_t max_length, const size_t max_backward, const size_t gap, - const BrotliEncoderParams* params, BackwardMatch* matches) { + const size_t max_length, const size_t max_backward, + const size_t gap, const BrotliEncoderParams* params, + BackwardMatch* matches) { BackwardMatch* const orig_matches = matches; const size_t cur_ix_masked = cur_ix & ring_buffer_mask; size_t best_len = 1; diff --git a/C/hashes/README.md b/C/hashes/README.md new file mode 100644 index 00000000..7d85680c --- /dev/null +++ b/C/hashes/README.md @@ -0,0 +1,4 @@ + +Taken from lib/crypto/* of samba-4.5.8.tar.gz. + +/TR 2018-11-15 diff --git a/C/zstd/README.md b/C/zstd/README.md index 0966c7ae..0c9cd6d5 100644 --- a/C/zstd/README.md +++ b/C/zstd/README.md @@ -7,13 +7,32 @@ in order to make it easier to select or exclude features. #### Building -`Makefile` script is provided, supporting all standard [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions), +`Makefile` script is provided, supporting [Makefile conventions](https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html#Makefile-Conventions), including commands variables, staged install, directory variables and standard targets. - `make` : generates both static and dynamic libraries -- `make install` : install libraries in default system directories +- `make install` : install libraries and headers in target system directories -`libzstd` default scope includes compression, decompression, dictionary building, -and decoding support for legacy formats >= v0.5.0. +`libzstd` default scope is pretty large, including compression, decompression, dictionary builder, +and support for decoding legacy formats >= v0.5.0. +The scope can be reduced on demand (see paragraph _modular build_). + + +#### Multithreading support + +Multithreading is disabled by default when building with `make`. +Enabling multithreading requires 2 conditions : +- set build macro `ZSTD_MULTITHREAD` (`-DZSTD_MULTITHREAD` for `gcc`) +- for POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) + +Both conditions are automatically applied when invoking `make lib-mt` target. + +When linking a POSIX program with a multithreaded version of `libzstd`, +note that it's necessary to request the `-pthread` flag during link stage. + +Multithreading capabilities are exposed +via the [advanced API defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/v1.3.8/lib/zstd.h#L592). +This API is still labelled experimental, +but is expected to become "stable" in the near future. #### API @@ -26,63 +45,70 @@ Zstandard's stable API is exposed within [lib/zstd.h](zstd.h). Optional advanced features are exposed via : - `lib/common/zstd_errors.h` : translates `size_t` function results - into an `ZSTD_ErrorCode`, for accurate error handling. + into a `ZSTD_ErrorCode`, for accurate error handling. + - `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`, - it unlocks access to advanced experimental API, - exposed in second part of `zstd.h`. - These APIs are not "stable", their definition may change in the future. - As a consequence, it shall ___never be used with dynamic library___ ! + it unlocks access to the experimental API, + exposed in the second part of `zstd.h`. + All definitions in the experimental APIs are unstable, + they may still change in the future, or even be removed. + As a consequence, experimental definitions shall ___never be used with dynamic library___ ! Only static linking is allowed. #### Modular build -It's possible to compile only a limited set of features. +It's possible to compile only a limited set of features within `libzstd`. +The file structure is designed to make this selection manually achievable for any build system : - Directory `lib/common` is always required, for all variants. + - Compression source code lies in `lib/compress` + - Decompression source code lies in `lib/decompress` + - It's possible to include only `compress` or only `decompress`, they don't depend on each other. + - `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples. The API is exposed in `lib/dictBuilder/zdict.h`. This module depends on both `lib/common` and `lib/compress` . -- `lib/legacy` : source code to decompress legacy zstd formats, starting from `v0.1.0`. + +- `lib/legacy` : makes it possible to decompress legacy zstd formats, starting from `v0.1.0`. This module depends on `lib/common` and `lib/decompress`. To enable this feature, define `ZSTD_LEGACY_SUPPORT` during compilation. Specifying a number limits versions supported to that version onward. For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats >= v0.2.0". - `ZSTD_LEGACY_SUPPORT=3` means : "support legacy formats >= v0.3.0", and so on. - Currently, the default library setting is `ZST_LEGACY_SUPPORT=5`. - It can be changed at build by any other value. - Note that any number >= 8 translates into "do __not__ support legacy formats", - since all versions of `zstd` >= v0.8 are compatible with v1+ specification. - `ZSTD_LEGACY_SUPPORT=0` also means "do __not__ support legacy formats". - Once enabled, this capability is transparently triggered within decompression functions. - It's also possible to invoke directly legacy API, as exposed in `lib/legacy/zstd_legacy.h`. - Each version also provides an additional dedicated set of advanced API. + Conversely, `ZSTD_LEGACY_SUPPORT=0` means "do __not__ support legacy formats". + By default, this build macro is set as `ZSTD_LEGACY_SUPPORT=5`. + Decoding supported legacy format is a transparent capability triggered within decompression functions. + It's also allowed to invoke legacy API directly, exposed in `lib/legacy/zstd_legacy.h`. + Each version does also provide its own set of advanced API. For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` . - Note : `lib/legacy` only supports _decoding_ legacy formats. -- Similarly, you can define `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, - and `ZSTD_LIB_DEPRECATED` as 0 to forgo compilation of the corresponding features. This will - also disable compilation of all dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable - dictBuilder). +- While invoking `make libzstd`, it's possible to define build macros + `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`, + and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features. + This will also disable compilation of all dependencies + (eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder). -#### Multithreading support +- There are some additional build macros that can be used to minify the decoder. -Multithreading is disabled by default when building with `make`. -Enabling multithreading requires 2 conditions : -- set macro `ZSTD_MULTITHREAD` -- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc`) + Zstandard often has more than one implementation of a piece of functionality, + where each implementation optimizes for different scenarios. For example, the + Huffman decoder has complementary implementations that decode the stream one + symbol at a time or two symbols at a time. Zstd normally includes both (and + dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` + or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding + compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` + and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of + only one or the other of two decompression implementations. The smallest + binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and + `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`. -Both conditions are automatically triggered by invoking `make lib-mt` target. -Note that, when linking a POSIX program with a multithreaded version of `libzstd`, -it's necessary to trigger `-pthread` flag during link stage. - -Multithreading capabilities are exposed -via [advanced API `ZSTD_compress_generic()` defined in `lib/zstd.h`](https://github.com/facebook/zstd/blob/dev/lib/zstd.h#L919). -This API is still considered experimental, -but is expected to become "stable" at some point in the future. + For squeezing the last ounce of size out, you can also define + `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`, + which removes the error messages that are otherwise returned by + `ZSTD_getErrorName`. #### Windows : using MinGW+MSYS to create DLL @@ -113,8 +139,8 @@ Consider migrating code towards supported streaming API exposed in `zstd.h`. The other files are not source code. There are : - - `LICENSE` : contains the BSD license text - - `Makefile` : `make` script to build and install zstd library (static and dynamic) - `BUCK` : support for `buck` build system (https://buckbuild.com/) - - `libzstd.pc.in` : for `pkg-config` (used in `make install`) + - `Makefile` : `make` script to build and install zstd library (static and dynamic) - `README.md` : this file + - `dll/` : resources directory for Windows compilation + - `libzstd.pc.in` : script for `pkg-config` (used in `make install`) diff --git a/C/zstd/bitstream.h b/C/zstd/bitstream.h index ef89b987..d955bd67 100644 --- a/C/zstd/bitstream.h +++ b/C/zstd/bitstream.h @@ -389,7 +389,7 @@ MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) * Read (consume) next n bits from local register and update. * Pay attention to not read more than nbBits contained into local register. * @return : extracted value. */ -MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) { size_t const value = BIT_lookBits(bitD, nbBits); BIT_skipBits(bitD, nbBits); @@ -398,7 +398,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) /*! BIT_readBitsFast() : * unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { size_t const value = BIT_lookBitsFast(bitD, nbBits); assert(nbBits >= 1); diff --git a/C/zstd/compiler.h b/C/zstd/compiler.h index 07f875e4..7f561282 100644 --- a/C/zstd/compiler.h +++ b/C/zstd/compiler.h @@ -15,6 +15,8 @@ * Compiler specifics *********************************************************/ /* force inlining */ + +#if !defined(ZSTD_NO_INLINE) #if defined (__GNUC__) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ # define INLINE_KEYWORD inline #else @@ -29,6 +31,13 @@ # define FORCE_INLINE_ATTR #endif +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + /** * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant * parameters. They must be inlined for the compiler to elimininate the constant @@ -89,23 +98,21 @@ #endif /* prefetch - * can be disabled, by declaring NO_PREFETCH macro - * All prefetch invocations use a single default locality 2, - * generating instruction prefetcht1, - * which, according to Intel, means "load data into L2 cache". - * This is a good enough "middle ground" for the time being, - * though in theory, it would be better to specialize locality depending on data being prefetched. - * Tests could not determine any sensible difference based on locality value. */ + * can be disabled, by declaring NO_PREFETCH build macro */ #if defined(NO_PREFETCH) -# define PREFETCH(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ #else # if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) /* _mm_prefetch() is not defined outside of x86/x64 */ # include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ -# define PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) -# define PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) # else -# define PREFETCH(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L1(ptr) (void)(ptr) /* disabled */ +# define PREFETCH_L2(ptr) (void)(ptr) /* disabled */ # endif #endif /* NO_PREFETCH */ @@ -116,7 +123,7 @@ size_t const _size = (size_t)(s); \ size_t _pos; \ for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ - PREFETCH(_ptr + _pos); \ + PREFETCH_L2(_ptr + _pos); \ } \ } diff --git a/C/zstd/cpu.h b/C/zstd/cpu.h index eeb428ad..5f0923fc 100644 --- a/C/zstd/cpu.h +++ b/C/zstd/cpu.h @@ -78,7 +78,7 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { __asm__( "pushl %%ebx\n\t" "cpuid\n\t" - "movl %%ebx, %%eax\n\r" + "movl %%ebx, %%eax\n\t" "popl %%ebx" : "=a"(f7b), "=c"(f7c) : "a"(7), "c"(0) diff --git a/C/zstd/debug.h b/C/zstd/debug.h index 0c04ad2c..b4fc89d4 100644 --- a/C/zstd/debug.h +++ b/C/zstd/debug.h @@ -57,9 +57,9 @@ extern "C" { #endif -/* static assert is triggered at compile time, leaving no runtime artefact, - * but can only work with compile-time constants. - * This variant can only be used inside a function. */ +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ #define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) @@ -70,9 +70,19 @@ extern "C" { # define DEBUGLEVEL 0 #endif + +/* DEBUGFILE can be defined externally, + * typically through compiler command line. + * note : currently useless. + * Value must be stderr or stdout */ +#ifndef DEBUGFILE +# define DEBUGFILE stderr +#endif + + /* recommended values for DEBUGLEVEL : - * 0 : no debug, all run-time functions disabled - * 1 : no display, enables assert() only + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display * 2 : reserved, for currently active debug path * 3 : events once per object lifetime (CCtx, CDict, etc.) * 4 : events once per frame @@ -81,7 +91,7 @@ extern "C" { * 7+: events at every position (*very* verbose) * * It's generally inconvenient to output traces > 5. - * In which case, it's possible to selectively enable higher verbosity levels + * In which case, it's possible to selectively trigger high verbosity levels * by modifying g_debug_level. */ @@ -95,11 +105,12 @@ extern "C" { #if (DEBUGLEVEL>=2) # include -extern int g_debuglevel; /* here, this variable is only declared, - it actually lives in debug.c, - and is shared by the whole process. - It's typically used to enable very verbose levels - on selective conditions (such as position in src) */ +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ # define RAWLOG(l, ...) { \ if (l<=g_debuglevel) { \ diff --git a/C/zstd/error_private.c b/C/zstd/error_private.c index d004ee63..7c1bb67a 100644 --- a/C/zstd/error_private.c +++ b/C/zstd/error_private.c @@ -14,6 +14,10 @@ const char* ERR_getErrorString(ERR_enum code) { +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else static const char* const notErrorCode = "Unspecified error code"; switch( code ) { @@ -39,10 +43,12 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(maxCode): default: return notErrorCode; } +#endif } diff --git a/C/zstd/fse.h b/C/zstd/fse.h index a5a6b6d4..f72c519b 100644 --- a/C/zstd/fse.h +++ b/C/zstd/fse.h @@ -512,7 +512,7 @@ MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) const U32 tableLog = MEM_read16(ptr); statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; - statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); statePtr->stateLog = tableLog; } @@ -531,7 +531,7 @@ MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U3 } } -MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) { FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; const U16* const stateTable = (const U16*)(statePtr->stateTable); diff --git a/C/zstd/fse_compress.c b/C/zstd/fse_compress.c index 4408f0ed..60f357bb 100644 --- a/C/zstd/fse_compress.c +++ b/C/zstd/fse_compress.c @@ -115,7 +115,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, /* symbol start positions */ { U32 u; cumul[0] = 0; - for (u=1; u<=maxSymbolValue+1; u++) { + for (u=1; u <= maxSymbolValue+1; u++) { if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ cumul[u] = cumul[u-1] + 1; tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); @@ -658,7 +658,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src BYTE* op = ostart; BYTE* const oend = ostart + dstSize; - U32 count[FSE_MAX_SYMBOL_VALUE+1]; + unsigned count[FSE_MAX_SYMBOL_VALUE+1]; S16 norm[FSE_MAX_SYMBOL_VALUE+1]; FSE_CTable* CTable = (FSE_CTable*)workSpace; size_t const CTableSize = FSE_CTABLE_SIZE_U32(tableLog, maxSymbolValue); @@ -672,7 +672,7 @@ size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t src if (!tableLog) tableLog = FSE_DEFAULT_TABLELOG; /* Scan input and build symbol stats */ - { CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, (unsigned*)scratchBuffer) ); + { CHECK_V_F(maxCount, HIST_count_wksp(count, &maxSymbolValue, src, srcSize, scratchBuffer, scratchBufferSize) ); if (maxCount == srcSize) return 1; /* only a single symbol in src : rle */ if (maxCount == 1) return 0; /* each symbol present maximum once => not compressible */ if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ diff --git a/C/zstd/hist.c b/C/zstd/hist.c index 16524756..45b7babc 100644 --- a/C/zstd/hist.c +++ b/C/zstd/hist.c @@ -73,6 +73,7 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, return largestCount; } +typedef enum { trustInput, checkMaxSymbolValue } HIST_checkInput_e; /* HIST_count_parallel_wksp() : * store histogram into 4 intermediate tables, recombined at the end. @@ -85,8 +86,8 @@ unsigned HIST_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, static size_t HIST_count_parallel_wksp( unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize, - unsigned checkMax, - unsigned* const workSpace) + HIST_checkInput_e check, + U32* const workSpace) { const BYTE* ip = (const BYTE*)source; const BYTE* const iend = ip+sourceSize; @@ -137,7 +138,7 @@ static size_t HIST_count_parallel_wksp( /* finish last symbols */ while (ipmaxSymbolValue; s--) { Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); @@ -157,14 +158,18 @@ static size_t HIST_count_parallel_wksp( /* HIST_countFast_wksp() : * Same as HIST_countFast(), but using an externally provided scratch buffer. - * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE + */ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize, - unsigned* workSpace) + void* workSpace, size_t workSpaceSize) { if (sourceSize < 1500) /* heuristic threshold */ return HIST_count_simple(count, maxSymbolValuePtr, source, sourceSize); - return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, trustInput, (U32*)workSpace); } /* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ @@ -172,24 +177,27 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* source, size_t sourceSize) { unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters); + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters, sizeof(tmpCounters)); } /* HIST_count_wksp() : * Same as HIST_count(), but using an externally provided scratch buffer. * `workSpace` size must be table of >= HIST_WKSP_SIZE_U32 unsigned */ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, - const void* source, size_t sourceSize, unsigned* workSpace) + const void* source, size_t sourceSize, + void* workSpace, size_t workSpaceSize) { + if ((size_t)workSpace & 3) return ERROR(GENERIC); /* must be aligned on 4-bytes boundaries */ + if (workSpaceSize < HIST_WKSP_SIZE) return ERROR(workSpace_tooSmall); if (*maxSymbolValuePtr < 255) - return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); + return HIST_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, checkMaxSymbolValue, (U32*)workSpace); *maxSymbolValuePtr = 255; - return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); + return HIST_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace, workSpaceSize); } size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize) { unsigned tmpCounters[HIST_WKSP_SIZE_U32]; - return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters); + return HIST_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters, sizeof(tmpCounters)); } diff --git a/C/zstd/hist.h b/C/zstd/hist.h index 8b1991a9..8b389358 100644 --- a/C/zstd/hist.h +++ b/C/zstd/hist.h @@ -41,11 +41,11 @@ /*! HIST_count(): * Provides the precise count of each byte within a table 'count'. - * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + * 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). * Updates *maxSymbolValuePtr with actual largest symbol value detected. - * @return : count of the most frequent symbol (which isn't identified). - * or an error code, which can be tested using HIST_isError(). - * note : if return == srcSize, there is only one symbol. + * @return : count of the most frequent symbol (which isn't identified). + * or an error code, which can be tested using HIST_isError(). + * note : if return == srcSize, there is only one symbol. */ size_t HIST_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); @@ -56,14 +56,16 @@ unsigned HIST_isError(size_t code); /**< tells if a return value is an error co /* --- advanced histogram functions --- */ #define HIST_WKSP_SIZE_U32 1024 +#define HIST_WKSP_SIZE (HIST_WKSP_SIZE_U32 * sizeof(unsigned)) /** HIST_count_wksp() : * Same as HIST_count(), but using an externally provided scratch buffer. * Benefit is this function will use very little stack space. - * `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32 + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE */ size_t HIST_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, - unsigned* workSpace); + void* workSpace, size_t workSpaceSize); /** HIST_countFast() : * same as HIST_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr. @@ -74,11 +76,12 @@ size_t HIST_countFast(unsigned* count, unsigned* maxSymbolValuePtr, /** HIST_countFast_wksp() : * Same as HIST_countFast(), but using an externally provided scratch buffer. - * `workSpace` must be a table of unsigned of size >= HIST_WKSP_SIZE_U32 + * `workSpace` is a writable buffer which must be 4-bytes aligned, + * `workSpaceSize` must be >= HIST_WKSP_SIZE */ size_t HIST_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, - unsigned* workSpace); + void* workSpace, size_t workSpaceSize); /*! HIST_count_simple() : * Same as HIST_countFast(), this function is unsafe, diff --git a/C/zstd/huf.h b/C/zstd/huf.h index de946411..6b572c44 100644 --- a/C/zstd/huf.h +++ b/C/zstd/huf.h @@ -173,15 +173,19 @@ typedef U32 HUF_DTable; * Advanced decompression functions ******************************************/ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ +#endif size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif /* **************************************** @@ -228,7 +232,7 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, #define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) size_t HUF_buildCTable_wksp (HUF_CElt* tree, - const U32* count, U32 maxSymbolValue, U32 maxNbBits, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); /*! HUF_readStats() : @@ -277,14 +281,22 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); #define HUF_DECOMPRESS_WORKSPACE_SIZE (2 << 10) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) +#ifndef HUF_FORCE_DECOMPRESS_X2 size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); +#endif size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#ifndef HUF_FORCE_DECOMPRESS_X2 size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif /* ====================== */ @@ -306,24 +318,36 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2); size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ +#endif size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); +#ifndef HUF_FORCE_DECOMPRESS_X2 size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ +#endif size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +#ifndef HUF_FORCE_DECOMPRESS_X2 size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +#endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +#ifndef HUF_FORCE_DECOMPRESS_X2 size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +#endif size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); diff --git a/C/zstd/huf_compress.c b/C/zstd/huf_compress.c index 4c40572f..f074f1e0 100644 --- a/C/zstd/huf_compress.c +++ b/C/zstd/huf_compress.c @@ -88,13 +88,13 @@ static size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weight BYTE* op = ostart; BYTE* const oend = ostart + dstSize; - U32 maxSymbolValue = HUF_TABLELOG_MAX; + unsigned maxSymbolValue = HUF_TABLELOG_MAX; U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; BYTE scratchBuffer[1< HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ @@ -664,11 +665,11 @@ static size_t HUF_compress_internal ( if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - singleStream, oldHufTable, bmi2); + nbStreams, oldHufTable, bmi2); } /* Scan input and build symbol stats */ - { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, table->count) ); + { CHECK_V_F(largest, HIST_count_wksp (table->count, &maxSymbolValue, (const BYTE*)src, srcSize, workSpace, wkspSize) ); if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ } @@ -683,14 +684,15 @@ static size_t HUF_compress_internal ( if (preferRepeat && repeat && *repeat != HUF_repeat_none) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - singleStream, oldHufTable, bmi2); + nbStreams, oldHufTable, bmi2); } /* Build Huffman Tree */ huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); - { CHECK_V_F(maxBits, HUF_buildCTable_wksp(table->CTable, table->count, - maxSymbolValue, huffLog, - table->nodeTable, sizeof(table->nodeTable)) ); + { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, + maxSymbolValue, huffLog, + table->nodeTable, sizeof(table->nodeTable)); + CHECK_F(maxBits); huffLog = (U32)maxBits; /* Zero unused symbols in CTable, so we can check it for validity */ memset(table->CTable + (maxSymbolValue + 1), 0, @@ -706,7 +708,7 @@ static size_t HUF_compress_internal ( if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - singleStream, oldHufTable, bmi2); + nbStreams, oldHufTable, bmi2); } } /* Use the new huffman table */ @@ -718,7 +720,7 @@ static size_t HUF_compress_internal ( } return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - singleStream, table->CTable, bmi2); + nbStreams, table->CTable, bmi2); } @@ -728,7 +730,7 @@ size_t HUF_compress1X_wksp (void* dst, size_t dstSize, void* workSpace, size_t wkspSize) { return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, 1 /*single stream*/, + maxSymbolValue, huffLog, HUF_singleStream, workSpace, wkspSize, NULL, NULL, 0, 0 /*bmi2*/); } @@ -740,7 +742,7 @@ size_t HUF_compress1X_repeat (void* dst, size_t dstSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) { return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, 1 /*single stream*/, + maxSymbolValue, huffLog, HUF_singleStream, workSpace, wkspSize, hufTable, repeat, preferRepeat, bmi2); } @@ -762,7 +764,7 @@ size_t HUF_compress4X_wksp (void* dst, size_t dstSize, void* workSpace, size_t wkspSize) { return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, 0 /*4 streams*/, + maxSymbolValue, huffLog, HUF_fourStreams, workSpace, wkspSize, NULL, NULL, 0, 0 /*bmi2*/); } @@ -777,7 +779,7 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize, HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2) { return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, 0 /* 4 streams */, + maxSymbolValue, huffLog, HUF_fourStreams, workSpace, wkspSize, hufTable, repeat, preferRepeat, bmi2); } diff --git a/C/zstd/huf_decompress.c b/C/zstd/huf_decompress.c index 83ecaff0..3f8bd297 100644 --- a/C/zstd/huf_decompress.c +++ b/C/zstd/huf_decompress.c @@ -43,6 +43,19 @@ #include "huf.h" #include "error_private.h" +/* ************************************************************** +* Macros +****************************************************************/ + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + /* ************************************************************** * Error Management @@ -58,6 +71,51 @@ #define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + if (bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + { \ + (void)bmi2; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + /*-***************************/ /* generic DTableDesc */ /*-***************************/ @@ -71,6 +129,8 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) } +#ifndef HUF_FORCE_DECOMPRESS_X2 + /*-***************************/ /* single-symbol decoding */ /*-***************************/ @@ -307,46 +367,6 @@ typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, const void *cSrc, size_t cSrcSize, const HUF_DTable *DTable); -#if DYNAMIC_BMI2 - -#define HUF_DGEN(fn) \ - \ - static size_t fn##_default( \ - void* dst, size_t dstSize, \ - const void* cSrc, size_t cSrcSize, \ - const HUF_DTable* DTable) \ - { \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - \ - static TARGET_ATTRIBUTE("bmi2") size_t fn##_bmi2( \ - void* dst, size_t dstSize, \ - const void* cSrc, size_t cSrcSize, \ - const HUF_DTable* DTable) \ - { \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - if (bmi2) { \ - return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ - } \ - return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ - } - -#else - -#define HUF_DGEN(fn) \ - static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ - { \ - (void)bmi2; \ - return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ - } - -#endif HUF_DGEN(HUF_decompress1X1_usingDTable_internal) HUF_DGEN(HUF_decompress4X1_usingDTable_internal) @@ -437,6 +457,10 @@ size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cS return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); } +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 /* *************************/ /* double-symbols decoding */ @@ -911,6 +935,8 @@ size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cS return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); } +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + /* ***********************************/ /* Universal decompression selectors */ @@ -921,8 +947,18 @@ size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const HUF_DTable* DTable) { DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif } size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, @@ -930,11 +966,22 @@ size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const HUF_DTable* DTable) { DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#else return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); +#endif } +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = { @@ -956,6 +1003,7 @@ static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, qu {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ }; +#endif /** HUF_selectDecoder() : * Tells which decoder is likely to decode faster, @@ -966,6 +1014,15 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) { assert(dstSize > 0); assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else /* decoder timing evaluation */ { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ U32 const D256 = (U32)(dstSize >> 8); @@ -973,14 +1030,18 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, to reduce cache eviction */ return DTime1 < DTime0; -} } + } +#endif +} typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) { +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; +#endif /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -989,7 +1050,17 @@ size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcS if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); +#else return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); +#endif } } @@ -1002,8 +1073,18 @@ size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); +#else return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; +#endif } } @@ -1025,8 +1106,19 @@ size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, if (cSrcSize == 0) return ERROR(corruption_detected); { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); - return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize): +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize): HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif } } @@ -1041,10 +1133,22 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize); +#else return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize): HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); +#endif } } @@ -1060,10 +1164,21 @@ size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) { DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif } +#ifndef HUF_FORCE_DECOMPRESS_X2 size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) { const BYTE* ip = (const BYTE*) cSrc; @@ -1075,12 +1190,23 @@ size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstS return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); } +#endif size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) { DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#else return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); +#endif } size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) @@ -1090,7 +1216,17 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds if (cSrcSize == 0) return ERROR(corruption_detected); { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#else return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); +#endif } } diff --git a/C/zstd/pool.c b/C/zstd/pool.c index 281b3824..7a829454 100644 --- a/C/zstd/pool.c +++ b/C/zstd/pool.c @@ -88,8 +88,8 @@ static void* POOL_thread(void* opaque) { ctx->numThreadsBusy++; ctx->queueEmpty = ctx->queueHead == ctx->queueTail; /* Unlock the mutex, signal a pusher, and run the job */ - ZSTD_pthread_mutex_unlock(&ctx->queueMutex); ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); job.function(job.opaque); diff --git a/C/zstd/zstd.h b/C/zstd/zstd.h index f2af4ac8..b18fc8a4 100644 --- a/C/zstd/zstd.h +++ b/C/zstd/zstd.h @@ -71,16 +71,16 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 3 -#define ZSTD_VERSION_RELEASE 7 +#define ZSTD_VERSION_RELEASE 8 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) -ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< useful to check dll version */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< to check runtime library version */ #define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE #define ZSTD_QUOTE(str) #str #define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) #define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) -ZSTDLIB_API const char* ZSTD_versionString(void); /* v1.3.0+ */ +ZSTDLIB_API const char* ZSTD_versionString(void); /* requires v1.3.0+ */ /*************************************** * Default constant @@ -110,7 +110,7 @@ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, const void* src, size_t compressedSize); -/*! ZSTD_getFrameContentSize() : added in v1.3.0 +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ * `src` should point to the start of a ZSTD encoded frame. * `srcSize` must be at least as large as the frame header. * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. @@ -167,8 +167,10 @@ ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /*! ZSTD_compressCCtx() : - * Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */ -ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx + * The function will compress at requested compression level, + * ignoring any other parameter */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); @@ -184,8 +186,11 @@ ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /*! ZSTD_decompressDCtx() : - * Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()) */ -ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters. + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); @@ -194,9 +199,12 @@ ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, * Simple dictionary API ***************************/ /*! ZSTD_compress_usingDict() : - * Compression using a predefined Dictionary (see dictBuilder/zdict.h). + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see dictBuilder/zdict.h). * Note : This function loads the dictionary, resulting in significant startup delay. - * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -204,9 +212,10 @@ ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, int compressionLevel); /*! ZSTD_decompress_usingDict() : - * Decompression using a predefined Dictionary (see dictBuilder/zdict.h). + * Decompression using a known Dictionary. * Dictionary must be identical to the one used during compression. * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, @@ -214,17 +223,18 @@ ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, const void* dict,size_t dictSize); -/********************************** +/*********************************** * Bulk processing dictionary API - *********************************/ + **********************************/ typedef struct ZSTD_CDict_s ZSTD_CDict; /*! ZSTD_createCDict() : - * When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. - * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. + * When compressing multiple messages / blocks using the same dictionary, it's recommended to load it only once. + * ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup cost. * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. - * `dictBuffer` can be released after ZSTD_CDict creation, since its content is copied within CDict - * Note : A ZSTD_CDict can be created with an empty dictionary, but it is inefficient for small data. */ + * `dictBuffer` can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate `dictBuffer` content. + * Note : A ZSTD_CDict can be created from an empty dictBuffer, but it is inefficient when used to compress small data. */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, int compressionLevel); @@ -234,11 +244,9 @@ ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); /*! ZSTD_compress_usingCDict() : * Compression using a digested Dictionary. - * Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. - * Note that compression level is decided during dictionary creation. - * Frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) - * Note : ZSTD_compress_usingCDict() can be used with a ZSTD_CDict created from an empty dictionary. - * But it is inefficient for small data, and it is recommended to use ZSTD_compressCCtx(). */ + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -249,7 +257,7 @@ typedef struct ZSTD_DDict_s ZSTD_DDict; /*! ZSTD_createDDict() : * Create a digested dictionary, ready to start decompression operation without startup delay. - * dictBuffer can be released after DDict creation, as its content is copied inside DDict */ + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); /*! ZSTD_freeDDict() : @@ -258,7 +266,7 @@ ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); /*! ZSTD_decompress_usingDDict() : * Decompression using a digested Dictionary. - * Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */ + * Recommended when same dictionary is used multiple times. */ ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -289,13 +297,17 @@ typedef struct ZSTD_outBuffer_s { * A ZSTD_CStream object is required to track streaming operation. * Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. * ZSTD_CStream objects can be reused multiple times on consecutive compression operations. -* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, -* since it will play nicer with system's memory, by re-using already allocated memory. -* Use one separate ZSTD_CStream per thread for parallel execution. +* It is recommended to re-use ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. * -* Start a new compression by initializing ZSTD_CStream context. -* Use ZSTD_initCStream() to start a new compression operation. -* Use variants ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for streaming with dictionary (experimental section) +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will re-use the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_initCStream() to set the parameter to a selected compression level. +* Use advanced API (ZSTD_CCtx_setParameter(), etc.) to set more specific parameters. * * Use ZSTD_compressStream() as many times as necessary to consume input stream. * The function will automatically update both `pos` fields within `input` and `output`. @@ -304,12 +316,11 @@ typedef struct ZSTD_outBuffer_s { * in which case `input.pos < input.size`. * The caller must check if input has been entirely consumed. * If not, the caller must make some room to receive more compressed data, -* typically by emptying output buffer, or allocating a new output buffer, * and then present again remaining input data. -* @return : a size hint, preferred nb of bytes to use as input for next function call -* or an error code, which can be tested using ZSTD_isError(). -* Note 1 : it's just a hint, to help latency a little, any other value will work fine. -* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() +* @return : a size hint, preferred nb of bytes to use as input for next function call +* or an error code, which can be tested using ZSTD_isError(). +* Note 1 : it's just a hint, to help latency a little, any value will work fine. +* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() * * At any moment, it's possible to flush whatever data might remain stuck within internal buffer, * using ZSTD_flushStream(). `output->pos` will be updated. @@ -353,25 +364,24 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. * ZSTD_DStream objects can be re-used multiple times. * -* Use ZSTD_initDStream() to start a new decompression operation, -* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. -* @return : recommended first input size +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. * * Use ZSTD_decompressStream() repetitively to consume your input. * The function will update both `pos` fields. * If `input.pos < input.size`, some input has not been consumed. * It's up to the caller to present again remaining data. -* The function tries to flush all data decoded immediately, repecting buffer sizes. +* The function tries to flush all data decoded immediately, respecting output buffer size. * If `output.pos < output.size`, decoder has flushed everything it could. -* But if `output.pos == output.size`, there is no such guarantee, -* it's likely that some decoded data was not flushed and still remains within internal buffers. +* But if `output.pos == output.size`, there might be some data left within internal buffers., * In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. -* When no additional input is provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. * @return : 0 when a frame is completely decoded and fully flushed, * or an error code, which can be tested using ZSTD_isError(), * or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : -* the return value is a suggested next input size (a hint for better latency) -* that will never load more than the current frame. +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. * *******************************************************************************/ typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ @@ -392,77 +402,602 @@ ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output -#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) -#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY - /**************************************************************************************** * ADVANCED AND EXPERIMENTAL FUNCTIONS **************************************************************************************** - * The definitions in this section are considered experimental. - * They should never be used with a dynamic library, as prototypes may change in the future. + * The definitions in the following section are considered experimental. * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. * Use them only in association with static linking. * ***************************************************************************************/ +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + + +/**************************************************************************************** + * Candidate API for promotion to stable status + **************************************************************************************** + * The following symbols and constants form the "staging area" : + * they are considered to join "stable API" by v1.4.0. + * The proposal is written so that it can be made stable "as is", + * though it's still possible to suggest improvements. + * Staging is in fact last chance for changes, + * the API is locked once reaching "stable" status. + * ***************************************************************************************/ + + +/* === Constants === */ + +/* all magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); -#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size to know frame header size */ + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + + +/*************************************** +* Advanced compression API +***************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * They do not apply to "simple" one-shot variants such as ZSTD_compressCCtx() + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supercedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove from experimental API entry points which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + + +typedef enum { + + /* compression parameters */ + ZSTD_c_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level sets all default values of other compression parameters */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such window size at decompression stage if using streaming. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless when using "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless when using "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming variants, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only useful if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * They return an error otherwise. */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression work is performed in parallel, within worker threads. + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or 1 MB, whichever is largest. + * The minimum size is automatically and transparently enforced */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001 +} ZSTD_cParameter; + + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overriden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +/*! ZSTD_CCtx_loadDictionary() : + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. + * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : + * Reference a prepared dictionary, to be used for all next compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supercede any compression parameter previously set within CCtx. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dm_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This removes any reference to any dictionary too. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + + + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize + * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it just acquires a copy of input, and distributes jobs to internal worker threads, flush whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + + +/* ============================== */ +/* Advanced decompression API */ +/* ============================== */ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_format + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000 + +} ZSTD_dParameter; + + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + + +/*! ZSTD_DCtx_loadDictionary() : + * Create an internal DDict from dict buffer, + * to be used to decompress next frames. + * The dictionary remains valid for all future frames, until explicitly invalidated. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX 5 /* minimum input size required to query frame header size */ #define ZSTD_FRAMEHEADERSIZE_MIN 6 -#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ -static const size_t ZSTD_frameHeaderSize_prefix = ZSTD_FRAMEHEADERSIZE_PREFIX; -static const size_t ZSTD_frameHeaderSize_min = ZSTD_FRAMEHEADERSIZE_MIN; -static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; -static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1<= first frame size - * @return : the compressed size of the first frame starting at `src`, - * suitable to pass to `ZSTD_decompress` or similar, - * or an error code if input is invalid */ -ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); - /*! ZSTD_findDecompressedSize() : * `src` should point the start of a series of ZSTD encoded and/or skippable frames * `srcSize` must be the _exact_ size of this series @@ -521,7 +1093,7 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize) ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); /*! ZSTD_frameHeaderSize() : - * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. * @return : size of the Frame Header, * or an error code (if srcSize is too small) */ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); @@ -531,16 +1103,6 @@ ZSTDLIB_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); * Memory management ***************************************/ -/*! ZSTD_sizeof_*() : - * These functions give the current memory usage of selected object. - * Object memory usage can evolve when re-used. */ -ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); -ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); -ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); -ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); -ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); -ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); - /*! ZSTD_estimate*() : * These functions make it possible to estimate memory usage * of a future {D,C}Ctx, before its creation. @@ -548,7 +1110,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCCtxSize_usingCParams() can provide a tighter estimation. * ZSTD_estimateCCtxSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1. + * ZSTD_estimateCCtxSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. * Note : CCtx size estimation is only correct for single-threaded compression. */ ZSTDLIB_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); @@ -560,7 +1122,7 @@ ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); * It will also consider src size to be arbitrarily "large", which is worst case. * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. - * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_p_nbWorkers is >= 1. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParam_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. * Note : CStream size estimation is only correct for single-threaded compression. * ZSTD_DStream memory budget depends on window Size. * This information can be passed manually, using ZSTD_estimateDStreamSize, @@ -623,6 +1185,7 @@ ZSTDLIB_API const ZSTD_DDict* ZSTD_initStaticDDict( ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + /*! Custom memory allocation : * These prototypes make it possible to pass your own allocation/free functions. * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. @@ -657,8 +1220,9 @@ ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictS /*! ZSTD_createCDict_byReference() : * Create a digested dictionary for compression - * Dictionary content is simply referenced, and therefore stays in dictBuffer. - * It is important that dictBuffer outlives CDict, it must remain read accessible throughout the lifetime of CDict */ + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. */ ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); /*! ZSTD_getCParams() : @@ -681,22 +1245,161 @@ ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); /*! ZSTD_compress_advanced() : -* Same as ZSTD_compress_usingDict(), with fine-tune control over each compression parameter */ -ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); + * Same as ZSTD_compress_usingDict(), with fine-tune control over compression parameters (by structure) */ +ZSTDLIB_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : -* Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ + * Same as ZSTD_compress_usingCDict(), with fine-tune control over frame parameters */ ZSTDLIB_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, ZSTD_frameParameters fParams); + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); -/*--- Advanced decompression functions ---*/ +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParam_setParameter() : + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParam_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ /*! ZSTD_isFrame() : * Tells if the content of `buffer` starts with a valid Frame Identifier. @@ -737,9 +1440,64 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 + +/*! ZSTD_DCtx_setFormat() : + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + /******************************************************************** * Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. ********************************************************************/ /*===== Advanced Streaming compression functions =====*/ @@ -751,7 +1509,7 @@ ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDi ZSTDLIB_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, unsigned long long pledgedSrcSize); /**< same as ZSTD_initCStream_usingCDict(), with control over frame parameters. pledgedSrcSize must be correct. If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. */ /*! ZSTD_resetCStream() : - * start a new compression job, using same parameters from previous job. + * start a new frame, using same parameters from previous frame. * This is typically useful to skip dictionary loading stage, since it will re-use it in-place. * Note that zcs must be init at least once before using ZSTD_resetCStream(). * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. @@ -790,16 +1548,13 @@ ZSTDLIB_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx * + there is no active job (could be checked with ZSTD_frameProgression()), or * + oldest job is still actively compressing data, * but everything it has produced has also been flushed so far, - * therefore flushing speed is currently limited by production speed of oldest job - * irrespective of the speed of concurrent newer jobs. + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. */ ZSTDLIB_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); - /*===== Advanced Streaming decompression functions =====*/ -typedef enum { DStream_p_maxWindowSize } ZSTD_DStreamParameter_e; -ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); /* obsolete : this API will be removed in a future version */ ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /**< note: no dictionary will be used if dict == NULL or dictSize < 8 */ ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict is referenced, it must outlive decompression session */ ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ @@ -940,12 +1695,17 @@ typedef struct { unsigned dictID; unsigned checksumFlag; } ZSTD_frameHeader; + /** ZSTD_getFrameHeader() : * decode Frame Header, or requires larger `srcSize`. * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, * or an error code, which can be tested using ZSTD_isError() */ ZSTDLIB_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); ZSTDLIB_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); @@ -962,526 +1722,6 @@ ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); -/* ============================================ */ -/** New advanced API (experimental) */ -/* ============================================ */ - -/* API design : - * In this advanced API, parameters are pushed one by one into an existing context, - * using ZSTD_CCtx_set*() functions. - * Pushed parameters are sticky : they are applied to next job, and any subsequent job. - * It's possible to reset parameters to "default" using ZSTD_CCtx_reset(). - * Important : "sticky" parameters only work with `ZSTD_compress_generic()` ! - * For any other entry point, "sticky" parameters are ignored ! - * - * This API is intended to replace all others advanced / experimental API entry points. - */ - -/* note on enum design : - * All enum will be pinned to explicit values before reaching "stable API" status */ - -typedef enum { - /* Opened question : should we have a format ZSTD_f_auto ? - * Today, it would mean exactly the same as ZSTD_f_zstd1. - * But, in the future, should several formats become supported, - * on the compression side, it would mean "default format". - * On the decompression side, it would mean "automatic format detection", - * so that ZSTD_f_zstd1 would mean "accept *only* zstd frames". - * Since meaning is a little different, another option could be to define different enums for compression and decompression. - * This question could be kept for later, when there are actually multiple formats to support, - * but there is also the question of pinning enum values, and pinning value `0` is especially important */ - ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ - ZSTD_f_zstd1_magicless, /* Variant of zstd frame format, without initial 4-bytes magic number. - * Useful to save 4 bytes per generated frame. - * Decoder cannot recognise automatically this format, requiring instructions. */ -} ZSTD_format_e; - -typedef enum { - /* compression format */ - ZSTD_p_format = 10, /* See ZSTD_format_e enum definition. - * Cast selected format as unsigned for ZSTD_CCtx_setParameter() compatibility. */ - - /* compression parameters */ - ZSTD_p_compressionLevel=100, /* Update all compression parameters according to pre-defined cLevel table - * Default level is ZSTD_CLEVEL_DEFAULT==3. - * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. - * Note 1 : it's possible to pass a negative compression level by casting it to unsigned type. - * Note 2 : setting a level sets all default values of other compression parameters. - * Note 3 : setting compressionLevel automatically updates ZSTD_p_compressLiterals. */ - ZSTD_p_windowLog, /* Maximum allowed back-reference distance, expressed as power of 2. - * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. - * Special: value 0 means "use default windowLog". - * Note: Using a window size greater than ZSTD_MAXWINDOWSIZE_DEFAULT (default: 2^27) - * requires explicitly allowing such window size during decompression stage. */ - ZSTD_p_hashLog, /* Size of the initial probe table, as a power of 2. - * Resulting table size is (1 << (hashLog+2)). - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. - * Larger tables improve compression ratio of strategies <= dFast, - * and improve speed of strategies > dFast. - * Special: value 0 means "use default hashLog". */ - ZSTD_p_chainLog, /* Size of the multi-probe search table, as a power of 2. - * Resulting table size is (1 << (chainLog+2)). - * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. - * Larger tables result in better and slower compression. - * This parameter is useless when using "fast" strategy. - * Note it's still useful when using "dfast" strategy, - * in which case it defines a secondary probe table. - * Special: value 0 means "use default chainLog". */ - ZSTD_p_searchLog, /* Number of search attempts, as a power of 2. - * More attempts result in better and slower compression. - * This parameter is useless when using "fast" and "dFast" strategies. - * Special: value 0 means "use default searchLog". */ - ZSTD_p_minMatch, /* Minimum size of searched matches (note : repCode matches can be smaller). - * Larger values make faster compression and decompression, but decrease ratio. - * Must be clamped between ZSTD_SEARCHLENGTH_MIN and ZSTD_SEARCHLENGTH_MAX. - * Note that currently, for all strategies < btopt, effective minimum is 4. - * , for all strategies > fast, effective maximum is 6. - * Special: value 0 means "use default minMatchLength". */ - ZSTD_p_targetLength, /* Impact of this field depends on strategy. - * For strategies btopt & btultra: - * Length of Match considered "good enough" to stop search. - * Larger values make compression stronger, and slower. - * For strategy fast: - * Distance between match sampling. - * Larger values make compression faster, and weaker. - * Special: value 0 means "use default targetLength". */ - ZSTD_p_compressionStrategy, /* See ZSTD_strategy enum definition. - * Cast selected strategy as unsigned for ZSTD_CCtx_setParameter() compatibility. - * The higher the value of selected strategy, the more complex it is, - * resulting in stronger and slower compression. - * Special: value 0 means "use default strategy". */ - - ZSTD_p_enableLongDistanceMatching=160, /* Enable long distance matching. - * This parameter is designed to improve compression ratio - * for large inputs, by finding large matches at long distance. - * It increases memory usage and window size. - * Note: enabling this parameter increases ZSTD_p_windowLog to 128 MB - * except when expressly set to a different value. */ - ZSTD_p_ldmHashLog, /* Size of the table for long distance matching, as a power of 2. - * Larger values increase memory usage and compression ratio, - * but decrease compression speed. - * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX - * default: windowlog - 7. - * Special: value 0 means "automatically determine hashlog". */ - ZSTD_p_ldmMinMatch, /* Minimum match size for long distance matcher. - * Larger/too small values usually decrease compression ratio. - * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. - * Special: value 0 means "use default value" (default: 64). */ - ZSTD_p_ldmBucketSizeLog, /* Log size of each bucket in the LDM hash table for collision resolution. - * Larger values improve collision resolution but decrease compression speed. - * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX . - * Special: value 0 means "use default value" (default: 3). */ - ZSTD_p_ldmHashEveryLog, /* Frequency of inserting/looking up entries in the LDM hash table. - * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). - * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. - * Larger values improve compression speed. - * Deviating far from default value will likely result in a compression ratio decrease. - * Special: value 0 means "automatically determine hashEveryLog". */ - - /* frame parameters */ - ZSTD_p_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) - * Content size must be known at the beginning of compression, - * it is provided using ZSTD_CCtx_setPledgedSrcSize() */ - ZSTD_p_checksumFlag, /* A 32-bits checksum of content is written at end of frame (default:0) */ - ZSTD_p_dictIDFlag, /* When applicable, dictionary's ID is written into frame header (default:1) */ - - /* multi-threading parameters */ - /* These parameters are only useful if multi-threading is enabled (ZSTD_MULTITHREAD). - * They return an error otherwise. */ - ZSTD_p_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. - * When nbWorkers >= 1, triggers asynchronous mode : - * ZSTD_compress_generic() consumes some input, flush some output if possible, and immediately gives back control to caller, - * while compression work is performed in parallel, within worker threads. - * (note : a strong exception to this rule is when first invocation sets ZSTD_e_end : it becomes a blocking call). - * More workers improve speed, but also increase memory usage. - * Default value is `0`, aka "single-threaded mode" : no worker is spawned, compression is performed inside Caller's thread, all invocations are blocking */ - ZSTD_p_jobSize, /* Size of a compression job. This value is enforced only in non-blocking mode. - * Each compression job is completed in parallel, so this value indirectly controls the nb of active threads. - * 0 means default, which is dynamically determined based on compression parameters. - * Job size must be a minimum of overlapSize, or 1 MB, whichever is largest. - * The minimum size is automatically and transparently enforced */ - ZSTD_p_overlapSizeLog, /* Size of previous input reloaded at the beginning of each job. - * 0 => no overlap, 6(default) => use 1/8th of windowSize, >=9 => use full windowSize */ - - /* =================================================================== */ - /* experimental parameters - no stability guaranteed */ - /* =================================================================== */ - - ZSTD_p_forceMaxWindow=1100, /* Force back-reference distances to remain < windowSize, - * even when referencing into Dictionary content (default:0) */ - ZSTD_p_forceAttachDict, /* ZSTD supports usage of a CDict in-place - * (avoiding having to copy the compression tables - * from the CDict into the working context). Using - * a CDict in this way saves an initial setup step, - * but comes at the cost of more work per byte of - * input. ZSTD has a simple internal heuristic that - * guesses which strategy will be faster. You can - * use this flag to override that guess. - * - * Note that the by-reference, in-place strategy is - * only used when reusing a compression context - * with compatible compression parameters. (If - * incompatible / uninitialized, the working - * context needs to be cleared anyways, which is - * about as expensive as overwriting it with the - * dictionary context, so there's no savings in - * using the CDict by-ref.) - * - * Values greater than 0 force attaching the dict. - * Values less than 0 force copying the dict. - * 0 selects the default heuristic-guided behavior. - */ - -} ZSTD_cParameter; - - -/*! ZSTD_CCtx_setParameter() : - * Set one compression parameter, selected by enum ZSTD_cParameter. - * Setting a parameter is generally only possible during frame initialization (before starting compression). - * Exception : when using multi-threading mode (nbThreads >= 1), - * following parameters can be updated _during_ compression (within same frame): - * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. - * new parameters will be active on next job, or after a flush(). - * Note : when `value` type is not unsigned (int, or enum), cast it to unsigned for proper type checking. - * @result : informational value (typically, value being set, correctly clamped), - * or an error code (which can be tested with ZSTD_isError()). */ -ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value); - -/*! ZSTD_CCtx_getParameter() : - * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value); - -/*! ZSTD_CCtx_setPledgedSrcSize() : - * Total input data size to be compressed as a single frame. - * This value will be controlled at the end, and result in error if not respected. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : 0 means zero, empty. - * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. - * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new compression job. - * Note 2 : If all data is provided and consumed in a single round, - * this value is overriden by srcSize instead. */ -ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); - -/*! ZSTD_CCtx_loadDictionary() : - * Create an internal CDict from `dict` buffer. - * Decompression will have to use same dictionary. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding a NULL (or 0-size) dictionary invalidates previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : Dictionary will be used for all future compression jobs. - * To return to "no-dictionary" situation, load a NULL dictionary - * Note 2 : Loading a dictionary involves building tables, which are dependent on compression parameters. - * For this reason, compression parameters cannot be changed anymore after loading a dictionary. - * It's also a CPU consuming operation, with non-negligible impact on latency. - * Note 3 :`dict` content will be copied internally. - * Use ZSTD_CCtx_loadDictionary_byReference() to reference dictionary content instead. - * In such a case, dictionary buffer must outlive its users. - * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - - -/*! ZSTD_CCtx_refCDict() : - * Reference a prepared dictionary, to be used for all next compression jobs. - * Note that compression parameters are enforced from within CDict, - * and supercede any compression parameter previously set within CCtx. - * The dictionary will remain valid for future compression jobs using same CCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : adding a NULL CDict means "return to no-dictionary mode". - * Note 1 : Currently, only one dictionary can be managed. - * Adding a new dictionary effectively "discards" any previous one. - * Note 2 : CDict is just referenced, its lifetime must outlive CCtx. */ -ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); - -/*! ZSTD_CCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compression job. - * Decompression will need same prefix to properly regenerate data. - * Compressing with a prefix is similar in outcome as performing a diff and compressing it, - * but performs much faster, especially during decompression (compression speed is tunable with compression level). - * Note that prefix is **only used once**. Tables are discarded at end of compression job (ZSTD_e_end). - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary - * Note 1 : Prefix buffer is referenced. It **must** outlive compression job. - * Its contain must remain unmodified up to end of compression (ZSTD_e_end). - * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, - * ensure that the window size is large enough to contain the entire source. - * See ZSTD_p_windowLog. - * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. - * It's a CPU consuming operation, with non-negligible impact on latency. - * If there is a need to use same prefix multiple times, consider loadDictionary instead. - * Note 4 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. */ -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize); -ZSTDLIB_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, - const void* prefix, size_t prefixSize, - ZSTD_dictContentType_e dictContentType); - -/*! ZSTD_CCtx_reset() : - * Return a CCtx to clean state. - * Useful after an error, or to interrupt an ongoing compression job and start a new one. - * Any internal data not yet flushed is cancelled. - * The parameters and dictionary are kept unchanged, to reset them use ZSTD_CCtx_resetParameters(). - */ -ZSTDLIB_API void ZSTD_CCtx_reset(ZSTD_CCtx* cctx); - -/*! ZSTD_CCtx_resetParameters() : - * All parameters are back to default values (compression level is ZSTD_CLEVEL_DEFAULT). - * Dictionary (if any) is dropped. - * Resetting parameters is only possible during frame initialization (before starting compression). - * To reset the context use ZSTD_CCtx_reset(). - * @return 0 or an error code (which can be checked with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx); - - - -typedef enum { - ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ - ZSTD_e_flush, /* flush any data provided so far, - * it creates (at least) one new block, that can be decoded immediately on reception; - * frame will continue: any future data can still reference previously compressed data, improving compression. */ - ZSTD_e_end /* flush any remaining data and close current frame. - * any additional data starts a new frame. - * each frame is independent (does not reference any content from previous frame). */ -} ZSTD_EndDirective; - -/*! ZSTD_compress_generic() : - * Behave about the same as ZSTD_compressStream. To note : - * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_setParameter() - * - Compression parameters cannot be changed once compression is started. - * - outpot->pos must be <= dstCapacity, input->pos must be <= srcSize - * - outpot->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. - * - In single-thread mode (default), function is blocking : it completed its job before returning to caller. - * - In multi-thread mode, function is non-blocking : it just acquires a copy of input, and distribute job to internal worker threads, - * and then immediately returns, just indicating that there is some data remaining to be flushed. - * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. - * - Exception : in multi-threading mode, if the first call requests a ZSTD_e_end directive, it is blocking : it will complete compression before giving back control to caller. - * - @return provides a minimum amount of data remaining to be flushed from internal buffers - * or an error code, which can be tested using ZSTD_isError(). - * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. - * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. - * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. - * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), - * only ZSTD_e_end or ZSTD_e_flush operations are allowed. - * Before starting a new compression job, or changing compression parameters, - * it is required to fully flush internal buffers. - */ -ZSTDLIB_API size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp); - - -/*! ZSTD_compress_generic_simpleArgs() : - * Same as ZSTD_compress_generic(), - * but using only integral types as arguments. - * Argument list is larger than ZSTD_{in,out}Buffer, - * but can be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_compress_generic_simpleArgs ( - ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos, - ZSTD_EndDirective endOp); - - -/*! ZSTD_CCtx_params : - * Quick howto : - * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure - * - ZSTD_CCtxParam_setParameter() : Push parameters one by one into - * an existing ZSTD_CCtx_params structure. - * This is similar to - * ZSTD_CCtx_setParameter(). - * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to - * an existing CCtx. - * These parameters will be applied to - * all subsequent compression jobs. - * - ZSTD_compress_generic() : Do compression using the CCtx. - * - ZSTD_freeCCtxParams() : Free the memory. - * - * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() - * for static allocation for single-threaded compression. - */ -ZSTDLIB_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); -ZSTDLIB_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); - - -/*! ZSTD_CCtxParams_reset() : - * Reset params to default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); - -/*! ZSTD_CCtxParams_init() : - * Initializes the compression parameters of cctxParams according to - * compression level. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); - -/*! ZSTD_CCtxParams_init_advanced() : - * Initializes the compression and frame parameters of cctxParams according to - * params. All other parameters are reset to their default values. - */ -ZSTDLIB_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); - - -/*! ZSTD_CCtxParam_setParameter() : - * Similar to ZSTD_CCtx_setParameter. - * Set one compression parameter, selected by enum ZSTD_cParameter. - * Parameters must be applied to a ZSTD_CCtx using ZSTD_CCtx_setParametersUsingCCtxParams(). - * Note : when `value` is an enum, cast it to unsigned for proper type checking. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned value); - -/*! ZSTD_CCtxParam_getParameter() : - * Similar to ZSTD_CCtx_getParameter. - * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_CCtxParam_getParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, unsigned* value); - -/*! ZSTD_CCtx_setParametersUsingCCtxParams() : - * Apply a set of ZSTD_CCtx_params to the compression context. - * This can be done even after compression is started, - * if nbWorkers==0, this will have no impact until a new compression is started. - * if nbWorkers>=1, new parameters will be picked up at next job, - * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). - */ -ZSTDLIB_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( - ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); - - -/* ==================================== */ -/*=== Advanced decompression API ===*/ -/* ==================================== */ - -/* The following API works the same way as the advanced compression API : - * a context is created, parameters are pushed into it one by one, - * then the context can be used to decompress data using an interface similar to the straming API. - */ - -/*! ZSTD_DCtx_loadDictionary() : - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, - * meaning "return to no-dictionary mode". - * Note 1 : `dict` content will be copied internally. - * Use ZSTD_DCtx_loadDictionary_byReference() - * to reference dictionary content instead. - * In which case, the dictionary buffer must outlive its users. - * Note 2 : Loading a dictionary involves building tables, - * which has a non-negligible impact on CPU usage and latency. - * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to select - * how dictionary content will be interpreted and loaded. - */ -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); -ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); - - -/*! ZSTD_DCtx_refDDict() : - * Reference a prepared dictionary, to be used to decompress next frames. - * The dictionary remains active for decompression of future frames using same DCtx. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. - * Special : adding a NULL DDict means "return to no-dictionary mode". - * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); - - -/*! ZSTD_DCtx_refPrefix() : - * Reference a prefix (single-usage dictionary) for next compression job. - * This is the reverse operation of ZSTD_CCtx_refPrefix(), - * and must use the same prefix as the one used during compression. - * Prefix is **only used once**. Reference is discarded at end of frame. - * End of frame is reached when ZSTD_DCtx_decompress_generic() returns 0. - * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary - * Note 2 : Prefix buffer is referenced. It **must** outlive decompression job. - * Prefix buffer must remain unmodified up to the end of frame, - * reached when ZSTD_DCtx_decompress_generic() returns 0. - * Note 3 : By default, the prefix is treated as raw content (ZSTD_dm_rawContent). - * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode. - * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. - * A fulldict prefix is more costly though. - */ -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, - const void* prefix, size_t prefixSize); -ZSTDLIB_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, - const void* prefix, size_t prefixSize, - ZSTD_dictContentType_e dictContentType); - - -/*! ZSTD_DCtx_setMaxWindowSize() : - * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. - * This is useful to prevent a decoder context from reserving too much memory for itself (potential attack scenario). - * This parameter is only useful in streaming mode, since no internal buffer is allocated in direct mode. - * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_MAX) - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); - - -/*! ZSTD_DCtx_setFormat() : - * Instruct the decoder context about what kind of data to decode next. - * This instruction is mandatory to decode data without a fully-formed header, - * such ZSTD_f_zstd1_magicless for example. - * @return : 0, or an error code (which can be tested using ZSTD_isError()). - */ -ZSTDLIB_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); - - -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, - const void* src, size_t srcSize, ZSTD_format_e format); - - -/*! ZSTD_decompress_generic() : - * Behave the same as ZSTD_decompressStream. - * Decompression parameters cannot be changed once decompression is started. - * @return : an error code, which can be tested using ZSTD_isError() - * if >0, a hint, nb of expected input bytes for next invocation. - * `0` means : a frame has just been fully decoded and flushed. - */ -ZSTDLIB_API size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input); - - -/*! ZSTD_decompress_generic_simpleArgs() : - * Same as ZSTD_decompress_generic(), - * but using only integral types as arguments. - * Argument list is larger than ZSTD_{in,out}Buffer, - * but can be helpful for binders from dynamic languages - * which have troubles handling structures containing memory pointers. - */ -ZSTDLIB_API size_t ZSTD_decompress_generic_simpleArgs ( - ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, size_t* dstPos, - const void* src, size_t srcSize, size_t* srcPos); - - -/*! ZSTD_DCtx_reset() : - * Return a DCtx to clean state. - * If a decompression was ongoing, any internal data not yet flushed is cancelled. - * All parameters are back to default values, including sticky ones. - * Dictionary (if any) is dropped. - * Parameters can be modified again after a reset. - */ -ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx); - - /* ============================ */ /** Block level API */ @@ -1501,10 +1741,10 @@ ZSTDLIB_API void ZSTD_DCtx_reset(ZSTD_DCtx* dctx); + copyCCtx() and copyDCtx() can be used too - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + If input is larger than a block size, it's necessary to split input data into multiple blocks - + For inputs larger than a single block size, consider using the regular ZSTD_compress() instead. + + For inputs larger than a single block, really consider using regular ZSTD_compress() instead. Frame metadata is not that costly, and quickly becomes negligible as source size grows larger. - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. - In which case, nothing is produced into `dst`. + In which case, nothing is produced into `dst` ! + User must test for such outcome and deal directly with uncompressed data + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + In case of multiple successive blocks, should some of them be uncompressed, diff --git a/C/zstd/zstd_common.c b/C/zstd/zstd_common.c index 6f05d240..667f4a27 100644 --- a/C/zstd/zstd_common.c +++ b/C/zstd/zstd_common.c @@ -30,8 +30,10 @@ const char* ZSTD_versionString(void) { return ZSTD_VERSION_STRING; } /*-**************************************** * ZSTD Error Management ******************************************/ +#undef ZSTD_isError /* defined within zstd_internal.h */ /*! ZSTD_isError() : - * tells if a return value is an error code */ + * tells if a return value is an error code + * symbol is required for external callers */ unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } /*! ZSTD_getErrorName() : diff --git a/C/zstd/zstd_compress.c b/C/zstd/zstd_compress.c index 5f6280a8..c2c9d3bc 100644 --- a/C/zstd/zstd_compress.c +++ b/C/zstd/zstd_compress.c @@ -11,6 +11,7 @@ /*-************************************* * Dependencies ***************************************/ +#include /* INT_MAX */ #include /* memset */ #include "cpu.h" #include "mem.h" @@ -61,7 +62,7 @@ static void ZSTD_initCCtx(ZSTD_CCtx* cctx, ZSTD_customMem memManager) memset(cctx, 0, sizeof(*cctx)); cctx->customMem = memManager; cctx->bmi2 = ZSTD_cpuid_bmi2(ZSTD_cpuid()); - { size_t const err = ZSTD_CCtx_resetParameters(cctx); + { size_t const err = ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); assert(!ZSTD_isError(err)); (void)err; } @@ -128,7 +129,7 @@ static size_t ZSTD_sizeof_mtctx(const ZSTD_CCtx* cctx) #ifdef ZSTD_MULTITHREAD return ZSTDMT_sizeof_CCtx(cctx->mtctx); #else - (void) cctx; + (void)cctx; return 0; #endif } @@ -226,9 +227,160 @@ static ZSTD_CCtx_params ZSTD_assignParamsToCCtxParams( return ret; } -#define CLAMPCHECK(val,min,max) { \ - if (((val)<(min)) | ((val)>(max))) { \ - return ERROR(parameter_outOfBound); \ +ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + + switch(param) + { + case ZSTD_c_compressionLevel: + bounds.lowerBound = ZSTD_minCLevel(); + bounds.upperBound = ZSTD_maxCLevel(); + return bounds; + + case ZSTD_c_windowLog: + bounds.lowerBound = ZSTD_WINDOWLOG_MIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + + case ZSTD_c_hashLog: + bounds.lowerBound = ZSTD_HASHLOG_MIN; + bounds.upperBound = ZSTD_HASHLOG_MAX; + return bounds; + + case ZSTD_c_chainLog: + bounds.lowerBound = ZSTD_CHAINLOG_MIN; + bounds.upperBound = ZSTD_CHAINLOG_MAX; + return bounds; + + case ZSTD_c_searchLog: + bounds.lowerBound = ZSTD_SEARCHLOG_MIN; + bounds.upperBound = ZSTD_SEARCHLOG_MAX; + return bounds; + + case ZSTD_c_minMatch: + bounds.lowerBound = ZSTD_MINMATCH_MIN; + bounds.upperBound = ZSTD_MINMATCH_MAX; + return bounds; + + case ZSTD_c_targetLength: + bounds.lowerBound = ZSTD_TARGETLENGTH_MIN; + bounds.upperBound = ZSTD_TARGETLENGTH_MAX; + return bounds; + + case ZSTD_c_strategy: + bounds.lowerBound = ZSTD_STRATEGY_MIN; + bounds.upperBound = ZSTD_STRATEGY_MAX; + return bounds; + + case ZSTD_c_contentSizeFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_checksumFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_dictIDFlag: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_nbWorkers: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_NBWORKERS_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_jobSize: + bounds.lowerBound = 0; +#ifdef ZSTD_MULTITHREAD + bounds.upperBound = ZSTDMT_JOBSIZE_MAX; +#else + bounds.upperBound = 0; +#endif + return bounds; + + case ZSTD_c_overlapLog: + bounds.lowerBound = ZSTD_OVERLAPLOG_MIN; + bounds.upperBound = ZSTD_OVERLAPLOG_MAX; + return bounds; + + case ZSTD_c_enableLongDistanceMatching: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_ldmHashLog: + bounds.lowerBound = ZSTD_LDM_HASHLOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHLOG_MAX; + return bounds; + + case ZSTD_c_ldmMinMatch: + bounds.lowerBound = ZSTD_LDM_MINMATCH_MIN; + bounds.upperBound = ZSTD_LDM_MINMATCH_MAX; + return bounds; + + case ZSTD_c_ldmBucketSizeLog: + bounds.lowerBound = ZSTD_LDM_BUCKETSIZELOG_MIN; + bounds.upperBound = ZSTD_LDM_BUCKETSIZELOG_MAX; + return bounds; + + case ZSTD_c_ldmHashRateLog: + bounds.lowerBound = ZSTD_LDM_HASHRATELOG_MIN; + bounds.upperBound = ZSTD_LDM_HASHRATELOG_MAX; + return bounds; + + /* experimental parameters */ + case ZSTD_c_rsyncable: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_forceMaxWindow : + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_format: + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + bounds.lowerBound = ZSTD_f_zstd1; + bounds.upperBound = ZSTD_f_zstd1_magicless; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + case ZSTD_c_forceAttachDict: + ZSTD_STATIC_ASSERT(ZSTD_dictDefaultAttach < ZSTD_dictForceCopy); + bounds.lowerBound = ZSTD_dictDefaultAttach; + bounds.upperBound = ZSTD_dictForceCopy; /* note : how to ensure at compile time that this is the highest value enum ? */ + return bounds; + + default: + { ZSTD_bounds const boundError = { ERROR(parameter_unsupported), 0, 0 }; + return boundError; + } + } +} + +/* ZSTD_cParam_withinBounds: + * @return 1 if value is within cParam bounds, + * 0 otherwise */ +static int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define BOUNDCHECK(cParam, val) { \ + if (!ZSTD_cParam_withinBounds(cParam,val)) { \ + return ERROR(parameter_outOfBound); \ } } @@ -236,38 +388,39 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) { switch(param) { - case ZSTD_p_compressionLevel: - case ZSTD_p_hashLog: - case ZSTD_p_chainLog: - case ZSTD_p_searchLog: - case ZSTD_p_minMatch: - case ZSTD_p_targetLength: - case ZSTD_p_compressionStrategy: + case ZSTD_c_compressionLevel: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: return 1; - case ZSTD_p_format: - case ZSTD_p_windowLog: - case ZSTD_p_contentSizeFlag: - case ZSTD_p_checksumFlag: - case ZSTD_p_dictIDFlag: - case ZSTD_p_forceMaxWindow : - case ZSTD_p_nbWorkers: - case ZSTD_p_jobSize: - case ZSTD_p_overlapSizeLog: - case ZSTD_p_enableLongDistanceMatching: - case ZSTD_p_ldmHashLog: - case ZSTD_p_ldmMinMatch: - case ZSTD_p_ldmBucketSizeLog: - case ZSTD_p_ldmHashEveryLog: - case ZSTD_p_forceAttachDict: + case ZSTD_c_format: + case ZSTD_c_windowLog: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: + case ZSTD_c_forceMaxWindow : + case ZSTD_c_nbWorkers: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: + case ZSTD_c_forceAttachDict: default: return 0; } } -size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned value) +size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) { - DEBUGLOG(4, "ZSTD_CCtx_setParameter (%u, %u)", (U32)param, value); + DEBUGLOG(4, "ZSTD_CCtx_setParameter (%i, %i)", (int)param, value); if (cctx->streamStage != zcss_init) { if (ZSTD_isUpdateAuthorized(param)) { cctx->cParamsChanged = 1; @@ -277,51 +430,52 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v switch(param) { - case ZSTD_p_format : + case ZSTD_c_format : return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_compressionLevel: + case ZSTD_c_compressionLevel: if (cctx->cdict) return ERROR(stage_wrong); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_windowLog: - case ZSTD_p_hashLog: - case ZSTD_p_chainLog: - case ZSTD_p_searchLog: - case ZSTD_p_minMatch: - case ZSTD_p_targetLength: - case ZSTD_p_compressionStrategy: + case ZSTD_c_windowLog: + case ZSTD_c_hashLog: + case ZSTD_c_chainLog: + case ZSTD_c_searchLog: + case ZSTD_c_minMatch: + case ZSTD_c_targetLength: + case ZSTD_c_strategy: if (cctx->cdict) return ERROR(stage_wrong); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_contentSizeFlag: - case ZSTD_p_checksumFlag: - case ZSTD_p_dictIDFlag: + case ZSTD_c_contentSizeFlag: + case ZSTD_c_checksumFlag: + case ZSTD_c_dictIDFlag: return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_forceMaxWindow : /* Force back-references to remain < windowSize, + case ZSTD_c_forceMaxWindow : /* Force back-references to remain < windowSize, * even when referencing into Dictionary content. * default : 0 when using a CDict, 1 when using a Prefix */ return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_forceAttachDict: + case ZSTD_c_forceAttachDict: return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_nbWorkers: - if ((value>0) && cctx->staticSize) { + case ZSTD_c_nbWorkers: + if ((value!=0) && cctx->staticSize) { return ERROR(parameter_unsupported); /* MT not compatible with static alloc */ } return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_jobSize: - case ZSTD_p_overlapSizeLog: + case ZSTD_c_jobSize: + case ZSTD_c_overlapLog: + case ZSTD_c_rsyncable: return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); - case ZSTD_p_enableLongDistanceMatching: - case ZSTD_p_ldmHashLog: - case ZSTD_p_ldmMinMatch: - case ZSTD_p_ldmBucketSizeLog: - case ZSTD_p_ldmHashEveryLog: + case ZSTD_c_enableLongDistanceMatching: + case ZSTD_c_ldmHashLog: + case ZSTD_c_ldmMinMatch: + case ZSTD_c_ldmBucketSizeLog: + case ZSTD_c_ldmHashRateLog: if (cctx->cdict) return ERROR(stage_wrong); return ZSTD_CCtxParam_setParameter(&cctx->requestedParams, param, value); @@ -329,21 +483,21 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned v } } -size_t ZSTD_CCtxParam_setParameter( - ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned value) +size_t ZSTD_CCtxParam_setParameter(ZSTD_CCtx_params* CCtxParams, + ZSTD_cParameter param, int value) { - DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%u, %u)", (U32)param, value); + DEBUGLOG(4, "ZSTD_CCtxParam_setParameter (%i, %i)", (int)param, value); switch(param) { - case ZSTD_p_format : - if (value > (unsigned)ZSTD_f_zstd1_magicless) - return ERROR(parameter_unsupported); + case ZSTD_c_format : + BOUNDCHECK(ZSTD_c_format, value); CCtxParams->format = (ZSTD_format_e)value; return (size_t)CCtxParams->format; - case ZSTD_p_compressionLevel : { - int cLevel = (int)value; /* cast expected to restore negative sign */ + case ZSTD_c_compressionLevel : { + int cLevel = value; if (cLevel > ZSTD_maxCLevel()) cLevel = ZSTD_maxCLevel(); + if (cLevel < ZSTD_minCLevel()) cLevel = ZSTD_minCLevel(); if (cLevel) { /* 0 : does not change current level */ CCtxParams->compressionLevel = cLevel; } @@ -351,213 +505,229 @@ size_t ZSTD_CCtxParam_setParameter( return 0; /* return type (size_t) cannot represent negative values */ } - case ZSTD_p_windowLog : - if (value>0) /* 0 => use default */ - CLAMPCHECK(value, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + case ZSTD_c_windowLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_windowLog, value); CCtxParams->cParams.windowLog = value; return CCtxParams->cParams.windowLog; - case ZSTD_p_hashLog : - if (value>0) /* 0 => use default */ - CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + case ZSTD_c_hashLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_hashLog, value); CCtxParams->cParams.hashLog = value; return CCtxParams->cParams.hashLog; - case ZSTD_p_chainLog : - if (value>0) /* 0 => use default */ - CLAMPCHECK(value, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + case ZSTD_c_chainLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_chainLog, value); CCtxParams->cParams.chainLog = value; return CCtxParams->cParams.chainLog; - case ZSTD_p_searchLog : - if (value>0) /* 0 => use default */ - CLAMPCHECK(value, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + case ZSTD_c_searchLog : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_searchLog, value); CCtxParams->cParams.searchLog = value; return value; - case ZSTD_p_minMatch : - if (value>0) /* 0 => use default */ - CLAMPCHECK(value, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - CCtxParams->cParams.searchLength = value; - return CCtxParams->cParams.searchLength; + case ZSTD_c_minMatch : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_minMatch, value); + CCtxParams->cParams.minMatch = value; + return CCtxParams->cParams.minMatch; - case ZSTD_p_targetLength : - /* all values are valid. 0 => use default */ + case ZSTD_c_targetLength : + BOUNDCHECK(ZSTD_c_targetLength, value); CCtxParams->cParams.targetLength = value; return CCtxParams->cParams.targetLength; - case ZSTD_p_compressionStrategy : - if (value>0) /* 0 => use default */ - CLAMPCHECK(value, (unsigned)ZSTD_fast, (unsigned)ZSTD_btultra); + case ZSTD_c_strategy : + if (value!=0) /* 0 => use default */ + BOUNDCHECK(ZSTD_c_strategy, value); CCtxParams->cParams.strategy = (ZSTD_strategy)value; return (size_t)CCtxParams->cParams.strategy; - case ZSTD_p_contentSizeFlag : + case ZSTD_c_contentSizeFlag : /* Content size written in frame header _when known_ (default:1) */ - DEBUGLOG(4, "set content size flag = %u", (value>0)); - CCtxParams->fParams.contentSizeFlag = value > 0; + DEBUGLOG(4, "set content size flag = %u", (value!=0)); + CCtxParams->fParams.contentSizeFlag = value != 0; return CCtxParams->fParams.contentSizeFlag; - case ZSTD_p_checksumFlag : + case ZSTD_c_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ - CCtxParams->fParams.checksumFlag = value > 0; + CCtxParams->fParams.checksumFlag = value != 0; return CCtxParams->fParams.checksumFlag; - case ZSTD_p_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ - DEBUGLOG(4, "set dictIDFlag = %u", (value>0)); + case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ + DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); CCtxParams->fParams.noDictIDFlag = !value; return !CCtxParams->fParams.noDictIDFlag; - case ZSTD_p_forceMaxWindow : - CCtxParams->forceWindow = (value > 0); + case ZSTD_c_forceMaxWindow : + CCtxParams->forceWindow = (value != 0); return CCtxParams->forceWindow; - case ZSTD_p_forceAttachDict : - CCtxParams->attachDictPref = value ? - (value > 0 ? ZSTD_dictForceAttach : ZSTD_dictForceCopy) : - ZSTD_dictDefaultAttach; + case ZSTD_c_forceAttachDict : { + const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; + BOUNDCHECK(ZSTD_c_forceAttachDict, pref); + CCtxParams->attachDictPref = pref; return CCtxParams->attachDictPref; + } - case ZSTD_p_nbWorkers : + case ZSTD_c_nbWorkers : #ifndef ZSTD_MULTITHREAD - if (value>0) return ERROR(parameter_unsupported); + if (value!=0) return ERROR(parameter_unsupported); return 0; #else return ZSTDMT_CCtxParam_setNbWorkers(CCtxParams, value); #endif - case ZSTD_p_jobSize : + case ZSTD_c_jobSize : #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_jobSize, value); #endif - case ZSTD_p_overlapSizeLog : + case ZSTD_c_overlapLog : #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapSectionLog, value); + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_overlapLog, value); #endif - case ZSTD_p_enableLongDistanceMatching : - CCtxParams->ldmParams.enableLdm = (value>0); + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + return ZSTDMT_CCtxParam_setMTCtxParameter(CCtxParams, ZSTDMT_p_rsyncable, value); +#endif + + case ZSTD_c_enableLongDistanceMatching : + CCtxParams->ldmParams.enableLdm = (value!=0); return CCtxParams->ldmParams.enableLdm; - case ZSTD_p_ldmHashLog : - if (value>0) /* 0 ==> auto */ - CLAMPCHECK(value, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + case ZSTD_c_ldmHashLog : + if (value!=0) /* 0 ==> auto */ + BOUNDCHECK(ZSTD_c_ldmHashLog, value); CCtxParams->ldmParams.hashLog = value; return CCtxParams->ldmParams.hashLog; - case ZSTD_p_ldmMinMatch : - if (value>0) /* 0 ==> default */ - CLAMPCHECK(value, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX); + case ZSTD_c_ldmMinMatch : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmMinMatch, value); CCtxParams->ldmParams.minMatchLength = value; return CCtxParams->ldmParams.minMatchLength; - case ZSTD_p_ldmBucketSizeLog : - if (value > ZSTD_LDM_BUCKETSIZELOG_MAX) - return ERROR(parameter_outOfBound); + case ZSTD_c_ldmBucketSizeLog : + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); CCtxParams->ldmParams.bucketSizeLog = value; return CCtxParams->ldmParams.bucketSizeLog; - case ZSTD_p_ldmHashEveryLog : + case ZSTD_c_ldmHashRateLog : if (value > ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN) return ERROR(parameter_outOfBound); - CCtxParams->ldmParams.hashEveryLog = value; - return CCtxParams->ldmParams.hashEveryLog; + CCtxParams->ldmParams.hashRateLog = value; + return CCtxParams->ldmParams.hashRateLog; default: return ERROR(parameter_unsupported); } } -size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, unsigned* value) +size_t ZSTD_CCtx_getParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value) { return ZSTD_CCtxParam_getParameter(&cctx->requestedParams, param, value); } size_t ZSTD_CCtxParam_getParameter( - ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, unsigned* value) + ZSTD_CCtx_params* CCtxParams, ZSTD_cParameter param, int* value) { switch(param) { - case ZSTD_p_format : + case ZSTD_c_format : *value = CCtxParams->format; break; - case ZSTD_p_compressionLevel : + case ZSTD_c_compressionLevel : *value = CCtxParams->compressionLevel; break; - case ZSTD_p_windowLog : + case ZSTD_c_windowLog : *value = CCtxParams->cParams.windowLog; break; - case ZSTD_p_hashLog : + case ZSTD_c_hashLog : *value = CCtxParams->cParams.hashLog; break; - case ZSTD_p_chainLog : + case ZSTD_c_chainLog : *value = CCtxParams->cParams.chainLog; break; - case ZSTD_p_searchLog : + case ZSTD_c_searchLog : *value = CCtxParams->cParams.searchLog; break; - case ZSTD_p_minMatch : - *value = CCtxParams->cParams.searchLength; + case ZSTD_c_minMatch : + *value = CCtxParams->cParams.minMatch; break; - case ZSTD_p_targetLength : + case ZSTD_c_targetLength : *value = CCtxParams->cParams.targetLength; break; - case ZSTD_p_compressionStrategy : + case ZSTD_c_strategy : *value = (unsigned)CCtxParams->cParams.strategy; break; - case ZSTD_p_contentSizeFlag : + case ZSTD_c_contentSizeFlag : *value = CCtxParams->fParams.contentSizeFlag; break; - case ZSTD_p_checksumFlag : + case ZSTD_c_checksumFlag : *value = CCtxParams->fParams.checksumFlag; break; - case ZSTD_p_dictIDFlag : + case ZSTD_c_dictIDFlag : *value = !CCtxParams->fParams.noDictIDFlag; break; - case ZSTD_p_forceMaxWindow : + case ZSTD_c_forceMaxWindow : *value = CCtxParams->forceWindow; break; - case ZSTD_p_forceAttachDict : + case ZSTD_c_forceAttachDict : *value = CCtxParams->attachDictPref; break; - case ZSTD_p_nbWorkers : + case ZSTD_c_nbWorkers : #ifndef ZSTD_MULTITHREAD assert(CCtxParams->nbWorkers == 0); #endif *value = CCtxParams->nbWorkers; break; - case ZSTD_p_jobSize : + case ZSTD_c_jobSize : #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - *value = CCtxParams->jobSize; + assert(CCtxParams->jobSize <= INT_MAX); + *value = (int)CCtxParams->jobSize; break; #endif - case ZSTD_p_overlapSizeLog : + case ZSTD_c_overlapLog : #ifndef ZSTD_MULTITHREAD return ERROR(parameter_unsupported); #else - *value = CCtxParams->overlapSizeLog; + *value = CCtxParams->overlapLog; break; #endif - case ZSTD_p_enableLongDistanceMatching : + case ZSTD_c_rsyncable : +#ifndef ZSTD_MULTITHREAD + return ERROR(parameter_unsupported); +#else + *value = CCtxParams->rsyncable; + break; +#endif + case ZSTD_c_enableLongDistanceMatching : *value = CCtxParams->ldmParams.enableLdm; break; - case ZSTD_p_ldmHashLog : + case ZSTD_c_ldmHashLog : *value = CCtxParams->ldmParams.hashLog; break; - case ZSTD_p_ldmMinMatch : + case ZSTD_c_ldmMinMatch : *value = CCtxParams->ldmParams.minMatchLength; break; - case ZSTD_p_ldmBucketSizeLog : + case ZSTD_c_ldmBucketSizeLog : *value = CCtxParams->ldmParams.bucketSizeLog; break; - case ZSTD_p_ldmHashEveryLog : - *value = CCtxParams->ldmParams.hashEveryLog; + case ZSTD_c_ldmHashRateLog : + *value = CCtxParams->ldmParams.hashRateLog; break; default: return ERROR(parameter_unsupported); } @@ -655,34 +825,35 @@ size_t ZSTD_CCtx_refPrefix_advanced( /*! ZSTD_CCtx_reset() : * Also dumps dictionary */ -void ZSTD_CCtx_reset(ZSTD_CCtx* cctx) +size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) { - cctx->streamStage = zcss_init; - cctx->pledgedSrcSizePlusOne = 0; + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + cctx->streamStage = zcss_init; + cctx->pledgedSrcSizePlusOne = 0; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); + cctx->cdict = NULL; + return ZSTD_CCtxParams_reset(&cctx->requestedParams); + } + return 0; } -size_t ZSTD_CCtx_resetParameters(ZSTD_CCtx* cctx) -{ - if (cctx->streamStage != zcss_init) return ERROR(stage_wrong); - cctx->cdict = NULL; - return ZSTD_CCtxParams_reset(&cctx->requestedParams); -} /** ZSTD_checkCParams() : control CParam values remain within authorized range. @return : 0, or an error code if one value is beyond authorized range */ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) { - CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMPCHECK(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0); - if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX) - return ERROR(parameter_outOfBound); - if ((U32)(cParams.strategy) > (U32)ZSTD_btultra) - return ERROR(parameter_unsupported); + BOUNDCHECK(ZSTD_c_windowLog, cParams.windowLog); + BOUNDCHECK(ZSTD_c_chainLog, cParams.chainLog); + BOUNDCHECK(ZSTD_c_hashLog, cParams.hashLog); + BOUNDCHECK(ZSTD_c_searchLog, cParams.searchLog); + BOUNDCHECK(ZSTD_c_minMatch, cParams.minMatch); + BOUNDCHECK(ZSTD_c_targetLength,cParams.targetLength); + BOUNDCHECK(ZSTD_c_strategy, cParams.strategy); return 0; } @@ -692,19 +863,19 @@ size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) static ZSTD_compressionParameters ZSTD_clampCParams(ZSTD_compressionParameters cParams) { -# define CLAMP(val,min,max) { \ - if (valmax) val=max; \ +# define CLAMP_TYPE(cParam, val, type) { \ + ZSTD_bounds const bounds = ZSTD_cParam_getBounds(cParam); \ + if ((int)valbounds.upperBound) val=(type)bounds.upperBound; \ } - CLAMP(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); - CLAMP(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); - CLAMP(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); - CLAMP(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); - CLAMP(cParams.searchLength, ZSTD_SEARCHLENGTH_MIN, ZSTD_SEARCHLENGTH_MAX); - ZSTD_STATIC_ASSERT(ZSTD_TARGETLENGTH_MIN == 0); - if (cParams.targetLength > ZSTD_TARGETLENGTH_MAX) - cParams.targetLength = ZSTD_TARGETLENGTH_MAX; - CLAMP(cParams.strategy, ZSTD_fast, ZSTD_btultra); +# define CLAMP(cParam, val) CLAMP_TYPE(cParam, val, int) + CLAMP(ZSTD_c_windowLog, cParams.windowLog); + CLAMP(ZSTD_c_chainLog, cParams.chainLog); + CLAMP(ZSTD_c_hashLog, cParams.hashLog); + CLAMP(ZSTD_c_searchLog, cParams.searchLog); + CLAMP(ZSTD_c_minMatch, cParams.minMatch); + CLAMP(ZSTD_c_targetLength,cParams.targetLength); + CLAMP_TYPE(ZSTD_c_strategy,cParams.strategy, ZSTD_strategy); return cParams; } @@ -774,7 +945,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( if (CCtxParams->cParams.hashLog) cParams.hashLog = CCtxParams->cParams.hashLog; if (CCtxParams->cParams.chainLog) cParams.chainLog = CCtxParams->cParams.chainLog; if (CCtxParams->cParams.searchLog) cParams.searchLog = CCtxParams->cParams.searchLog; - if (CCtxParams->cParams.searchLength) cParams.searchLength = CCtxParams->cParams.searchLength; + if (CCtxParams->cParams.minMatch) cParams.minMatch = CCtxParams->cParams.minMatch; if (CCtxParams->cParams.targetLength) cParams.targetLength = CCtxParams->cParams.targetLength; if (CCtxParams->cParams.strategy) cParams.strategy = CCtxParams->cParams.strategy; assert(!ZSTD_checkCParams(cParams)); @@ -787,13 +958,12 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->searchLength==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); size_t const optPotentialSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<strategy == ZSTD_btopt) || - (cParams->strategy == ZSTD_btultra))) + size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) ? optPotentialSpace : 0; DEBUGLOG(4, "chainSize: %u - hSize: %u - h3Size: %u", @@ -808,7 +978,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, 0, 0); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); - U32 const divider = (cParams.searchLength==3) ? 3 : 4; + U32 const divider = (cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; size_t const entropySpace = HUF_WORKSPACE_SIZE; @@ -843,7 +1013,7 @@ size_t ZSTD_estimateCCtxSize(int compressionLevel) { int level; size_t memBudget = 0; - for (level=1; level<=compressionLevel; level++) { + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { size_t const newMB = ZSTD_estimateCCtxSize_internal(level); if (newMB > memBudget) memBudget = newMB; } @@ -879,7 +1049,7 @@ size_t ZSTD_estimateCStreamSize(int compressionLevel) { int level; size_t memBudget = 0; - for (level=1; level<=compressionLevel; level++) { + for (level=MIN(compressionLevel, 1); level<=compressionLevel; level++) { size_t const newMB = ZSTD_estimateCStreamSize_internal(level); if (newMB > memBudget) memBudget = newMB; } @@ -933,7 +1103,7 @@ static U32 ZSTD_equivalentCParams(ZSTD_compressionParameters cParams1, return (cParams1.hashLog == cParams2.hashLog) & (cParams1.chainLog == cParams2.chainLog) & (cParams1.strategy == cParams2.strategy) /* opt parser space */ - & ((cParams1.searchLength==3) == (cParams2.searchLength==3)); /* hashlog3 space */ + & ((cParams1.minMatch==3) == (cParams2.minMatch==3)); /* hashlog3 space */ } static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, @@ -945,7 +1115,7 @@ static void ZSTD_assertEqualCParams(ZSTD_compressionParameters cParams1, assert(cParams1.chainLog == cParams2.chainLog); assert(cParams1.hashLog == cParams2.hashLog); assert(cParams1.searchLog == cParams2.searchLog); - assert(cParams1.searchLength == cParams2.searchLength); + assert(cParams1.minMatch == cParams2.minMatch); assert(cParams1.targetLength == cParams2.targetLength); assert(cParams1.strategy == cParams2.strategy); } @@ -960,7 +1130,7 @@ static U32 ZSTD_equivalentLdmParams(ldmParams_t ldmParams1, ldmParams1.hashLog == ldmParams2.hashLog && ldmParams1.bucketSizeLog == ldmParams2.bucketSizeLog && ldmParams1.minMatchLength == ldmParams2.minMatchLength && - ldmParams1.hashEveryLog == ldmParams2.hashEveryLog); + ldmParams1.hashRateLog == ldmParams2.hashRateLog); } typedef enum { ZSTDb_not_buffered, ZSTDb_buffered } ZSTD_buffered_policy_e; @@ -976,7 +1146,7 @@ static U32 ZSTD_sufficientBuff(size_t bufferSize1, size_t maxNbSeq1, { size_t const windowSize2 = MAX(1, (size_t)MIN(((U64)1 << cParams2.windowLog), pledgedSrcSize)); size_t const blockSize2 = MIN(ZSTD_BLOCKSIZE_MAX, windowSize2); - size_t const maxNbSeq2 = blockSize2 / ((cParams2.searchLength == 3) ? 3 : 4); + size_t const maxNbSeq2 = blockSize2 / ((cParams2.minMatch == 3) ? 3 : 4); size_t const maxNbLit2 = blockSize2; size_t const neededBufferSize2 = (buffPol2==ZSTDb_buffered) ? windowSize2 + blockSize2 : 0; DEBUGLOG(4, "ZSTD_sufficientBuff: is neededBufferSize2=%u <= bufferSize1=%u", @@ -1034,8 +1204,8 @@ static void ZSTD_invalidateMatchState(ZSTD_matchState_t* ms) { ZSTD_window_clear(&ms->window); - ms->nextToUpdate = ms->window.dictLimit + 1; - ms->nextToUpdate3 = ms->window.dictLimit + 1; + ms->nextToUpdate = ms->window.dictLimit; + ms->nextToUpdate3 = ms->window.dictLimit; ms->loadedDictEnd = 0; ms->opt.litLengthSum = 0; /* force reset of btopt stats */ ms->dictMatchState = NULL; @@ -1080,7 +1250,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, { size_t const chainSize = (cParams->strategy == ZSTD_fast) ? 0 : ((size_t)1 << cParams->chainLog); size_t const hSize = ((size_t)1) << cParams->hashLog; - U32 const hashLog3 = (forCCtx && cParams->searchLength==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; + U32 const hashLog3 = (forCCtx && cParams->minMatch==3) ? MIN(ZSTD_HASHLOG3_MAX, cParams->windowLog) : 0; size_t const h3Size = ((size_t)1) << hashLog3; size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); @@ -1094,9 +1264,9 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_invalidateMatchState(ms); /* opt parser space */ - if (forCCtx && ((cParams->strategy == ZSTD_btopt) | (cParams->strategy == ZSTD_btultra))) { + if (forCCtx && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); - ms->opt.litFreq = (U32*)ptr; + ms->opt.litFreq = (unsigned*)ptr; ms->opt.litLengthFreq = ms->opt.litFreq + (1<opt.matchLengthFreq = ms->opt.litLengthFreq + (MaxLL+1); ms->opt.offCodeFreq = ms->opt.matchLengthFreq + (MaxML+1); @@ -1158,13 +1328,13 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* Adjust long distance matching parameters */ ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashEveryLog < 32); - zc->ldmState.hashPower = ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); + assert(params.ldmParams.hashRateLog < 32); + zc->ldmState.hashPower = ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength); } { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params.cParams.windowLog), pledgedSrcSize)); size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; + U32 const divider = (params.cParams.minMatch==3) ? 3 : 4; size_t const maxNbSeq = blockSize / divider; size_t const tokenSpace = WILDCOPY_OVERLENGTH + blockSize + 11*maxNbSeq; size_t const buffOutSize = (zbuff==ZSTDb_buffered) ? ZSTD_compressBound(blockSize)+1 : 0; @@ -1227,7 +1397,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, if (pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN) zc->appliedParams.fParams.contentSizeFlag = 0; DEBUGLOG(4, "pledged content size : %u ; flag : %u", - (U32)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); + (unsigned)pledgedSrcSize, zc->appliedParams.fParams.contentSizeFlag); zc->blockSize = blockSize; XXH64_reset(&zc->xxhState, 0); @@ -1306,16 +1476,17 @@ void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx) { * dictionary tables into the working context is faster than using them * in-place. */ -static const size_t attachDictSizeCutoffs[(unsigned)ZSTD_btultra+1] = { - 8 KB, /* unused */ - 8 KB, /* ZSTD_fast */ +static const size_t attachDictSizeCutoffs[ZSTD_STRATEGY_MAX+1] = { + 8 KB, /* unused */ + 8 KB, /* ZSTD_fast */ 16 KB, /* ZSTD_dfast */ 32 KB, /* ZSTD_greedy */ 32 KB, /* ZSTD_lazy */ 32 KB, /* ZSTD_lazy2 */ 32 KB, /* ZSTD_btlazy2 */ 32 KB, /* ZSTD_btopt */ - 8 KB /* ZSTD_btultra */ + 8 KB, /* ZSTD_btultra */ + 8 KB /* ZSTD_btultra2 */ }; static int ZSTD_shouldAttachDict(const ZSTD_CDict* cdict, @@ -1447,7 +1618,8 @@ static size_t ZSTD_resetCCtx_usingCDict(ZSTD_CCtx* cctx, ZSTD_buffered_policy_e zbuff) { - DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", (U32)pledgedSrcSize); + DEBUGLOG(4, "ZSTD_resetCCtx_usingCDict (pledgedSrcSize=%u)", + (unsigned)pledgedSrcSize); if (ZSTD_shouldAttachDict(cdict, params, pledgedSrcSize)) { return ZSTD_resetCCtx_byAttachingCDict( @@ -1670,7 +1842,9 @@ static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, cons * note : use same formula for both situations */ static size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) { - U32 const minlog = (strat==ZSTD_btultra) ? 7 : 6; + U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; + ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); return (srcSize >> minlog) + 2; } @@ -1679,7 +1853,8 @@ static size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, ZSTD_strategy strategy, int disableLiteralCompression, void* dst, size_t dstCapacity, const void* src, size_t srcSize, - U32* workspace, const int bmi2) + void* workspace, size_t wkspSize, + const int bmi2) { size_t const minGain = ZSTD_minGain(srcSize, strategy); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); @@ -1708,9 +1883,9 @@ static size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; cLitSize = singleStream ? HUF_compress1X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, HUF_WORKSPACE_SIZE, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) + workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2) : HUF_compress4X_repeat(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, - workspace, HUF_WORKSPACE_SIZE, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); + workspace, wkspSize, (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2); if (repeat != HUF_repeat_none) { /* reused the existing table */ hType = set_repeat; @@ -1977,7 +2152,7 @@ ZSTD_selectEncodingType( assert(!ZSTD_isError(NCountCost)); assert(compressedCost < ERROR(maxCode)); DEBUGLOG(5, "Estimated bit costs: basic=%u\trepeat=%u\tcompressed=%u", - (U32)basicCost, (U32)repeatCost, (U32)compressedCost); + (unsigned)basicCost, (unsigned)repeatCost, (unsigned)compressedCost); if (basicCost <= repeatCost && basicCost <= compressedCost) { DEBUGLOG(5, "Selected set_basic"); assert(isDefaultAllowed); @@ -1999,7 +2174,7 @@ ZSTD_selectEncodingType( MEM_STATIC size_t ZSTD_buildCTable(void* dst, size_t dstCapacity, FSE_CTable* nextCTable, U32 FSELog, symbolEncodingType_e type, - U32* count, U32 max, + unsigned* count, U32 max, const BYTE* codeTable, size_t nbSeq, const S16* defaultNorm, U32 defaultNormLog, U32 defaultMax, const FSE_CTable* prevCTable, size_t prevCTableSize, @@ -2007,11 +2182,13 @@ ZSTD_buildCTable(void* dst, size_t dstCapacity, { BYTE* op = (BYTE*)dst; const BYTE* const oend = op + dstCapacity; + DEBUGLOG(6, "ZSTD_buildCTable (dstCapacity=%u)", (unsigned)dstCapacity); switch (type) { case set_rle: - *op = codeTable[0]; CHECK_F(FSE_buildCTable_rle(nextCTable, (BYTE)max)); + if (dstCapacity==0) return ERROR(dstSize_tooSmall); + *op = codeTable[0]; return 1; case set_repeat: memcpy(nextCTable, prevCTable, prevCTableSize); @@ -2053,6 +2230,9 @@ ZSTD_encodeSequences_body( FSE_CState_t stateLitLength; CHECK_E(BIT_initCStream(&blockStream, dst, dstCapacity), dstSize_tooSmall); /* not enough space remaining */ + DEBUGLOG(6, "available space for bitstream : %i (dstCapacity=%u)", + (int)(blockStream.endPtr - blockStream.startPtr), + (unsigned)dstCapacity); /* first symbols */ FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); @@ -2085,9 +2265,9 @@ ZSTD_encodeSequences_body( U32 const ofBits = ofCode; U32 const mlBits = ML_bits[mlCode]; DEBUGLOG(6, "encoding: litlen:%2u - matchlen:%2u - offCode:%7u", - sequences[n].litLength, - sequences[n].matchLength + MINMATCH, - sequences[n].offset); + (unsigned)sequences[n].litLength, + (unsigned)sequences[n].matchLength + MINMATCH, + (unsigned)sequences[n].offset); /* 32b*/ /* 64b*/ /* (7)*/ /* (7)*/ FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ @@ -2112,6 +2292,7 @@ ZSTD_encodeSequences_body( BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ } BIT_flushBits(&blockStream); /* (7)*/ + DEBUGLOG(7, "remaining space : %i", (int)(blockStream.endPtr - blockStream.ptr)); } } DEBUGLOG(6, "ZSTD_encodeSequences: flushing ML state with %u bits", stateMatchLength.stateLog); @@ -2169,6 +2350,7 @@ static size_t ZSTD_encodeSequences( FSE_CTable const* CTable_LitLength, BYTE const* llCodeTable, seqDef const* sequences, size_t nbSeq, int longOffsets, int bmi2) { + DEBUGLOG(5, "ZSTD_encodeSequences: dstCapacity = %u", (unsigned)dstCapacity); #if DYNAMIC_BMI2 if (bmi2) { return ZSTD_encodeSequences_bmi2(dst, dstCapacity, @@ -2186,16 +2368,20 @@ static size_t ZSTD_encodeSequences( sequences, nbSeq, longOffsets); } -MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, - ZSTD_entropyCTables_t const* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - ZSTD_CCtx_params const* cctxParams, - void* dst, size_t dstCapacity, U32* workspace, - const int bmi2) +/* ZSTD_compressSequences_internal(): + * actually compresses both literals and sequences */ +MEM_STATIC size_t +ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* workspace, size_t wkspSize, + const int bmi2) { const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; ZSTD_strategy const strategy = cctxParams->cParams.strategy; - U32 count[MaxSeq+1]; + unsigned count[MaxSeq+1]; FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; @@ -2212,6 +2398,7 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, BYTE* lastNCount = NULL; ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<litStart; @@ -2222,7 +2409,8 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, cctxParams->cParams.strategy, disableLiteralCompression, op, dstCapacity, literals, litSize, - workspace, bmi2); + workspace, wkspSize, + bmi2); if (ZSTD_isError(cSize)) return cSize; assert(cSize <= dstCapacity); @@ -2249,51 +2437,63 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, /* convert length/distances into codes */ ZSTD_seqToCodes(seqStorePtr); /* build CTable for Literal Lengths */ - { U32 max = MaxLL; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace); /* can't fail */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ DEBUGLOG(5, "Building LL table"); nextEntropy->fse.litlength_repeatMode = prevEntropy->fse.litlength_repeatMode; - LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, count, max, mostFrequent, nbSeq, LLFSELog, prevEntropy->fse.litlengthCTable, LL_defaultNorm, LL_defaultNormLog, ZSTD_defaultAllowed, strategy); + LLtype = ZSTD_selectEncodingType(&nextEntropy->fse.litlength_repeatMode, + count, max, mostFrequent, nbSeq, + LLFSELog, prevEntropy->fse.litlengthCTable, + LL_defaultNorm, LL_defaultNormLog, + ZSTD_defaultAllowed, strategy); assert(set_basic < set_compressed && set_rle < set_compressed); assert(!(LLtype < set_compressed && nextEntropy->fse.litlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_LitLength, LLFSELog, (symbolEncodingType_e)LLtype, count, max, llCodeTable, nbSeq, LL_defaultNorm, LL_defaultNormLog, MaxLL, prevEntropy->fse.litlengthCTable, sizeof(prevEntropy->fse.litlengthCTable), - workspace, HUF_WORKSPACE_SIZE); + workspace, wkspSize); if (ZSTD_isError(countSize)) return countSize; if (LLtype == set_compressed) lastNCount = op; op += countSize; } } /* build CTable for Offsets */ - { U32 max = MaxOff; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace); /* can't fail */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ /* We can only use the basic table if max <= DefaultMaxOff, otherwise the offsets are too large */ ZSTD_defaultPolicy_e const defaultPolicy = (max <= DefaultMaxOff) ? ZSTD_defaultAllowed : ZSTD_defaultDisallowed; DEBUGLOG(5, "Building OF table"); nextEntropy->fse.offcode_repeatMode = prevEntropy->fse.offcode_repeatMode; - Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, count, max, mostFrequent, nbSeq, OffFSELog, prevEntropy->fse.offcodeCTable, OF_defaultNorm, OF_defaultNormLog, defaultPolicy, strategy); + Offtype = ZSTD_selectEncodingType(&nextEntropy->fse.offcode_repeatMode, + count, max, mostFrequent, nbSeq, + OffFSELog, prevEntropy->fse.offcodeCTable, + OF_defaultNorm, OF_defaultNormLog, + defaultPolicy, strategy); assert(!(Offtype < set_compressed && nextEntropy->fse.offcode_repeatMode != FSE_repeat_none)); /* We don't copy tables */ { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_OffsetBits, OffFSELog, (symbolEncodingType_e)Offtype, count, max, ofCodeTable, nbSeq, OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, prevEntropy->fse.offcodeCTable, sizeof(prevEntropy->fse.offcodeCTable), - workspace, HUF_WORKSPACE_SIZE); + workspace, wkspSize); if (ZSTD_isError(countSize)) return countSize; if (Offtype == set_compressed) lastNCount = op; op += countSize; } } /* build CTable for MatchLengths */ - { U32 max = MaxML; - size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace); /* can't fail */ - DEBUGLOG(5, "Building ML table"); + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, workspace, wkspSize); /* can't fail */ + DEBUGLOG(5, "Building ML table (remaining space : %i)", (int)(oend-op)); nextEntropy->fse.matchlength_repeatMode = prevEntropy->fse.matchlength_repeatMode; - MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, count, max, mostFrequent, nbSeq, MLFSELog, prevEntropy->fse.matchlengthCTable, ML_defaultNorm, ML_defaultNormLog, ZSTD_defaultAllowed, strategy); + MLtype = ZSTD_selectEncodingType(&nextEntropy->fse.matchlength_repeatMode, + count, max, mostFrequent, nbSeq, + MLFSELog, prevEntropy->fse.matchlengthCTable, + ML_defaultNorm, ML_defaultNormLog, + ZSTD_defaultAllowed, strategy); assert(!(MLtype < set_compressed && nextEntropy->fse.matchlength_repeatMode != FSE_repeat_none)); /* We don't copy tables */ { size_t const countSize = ZSTD_buildCTable(op, oend - op, CTable_MatchLength, MLFSELog, (symbolEncodingType_e)MLtype, count, max, mlCodeTable, nbSeq, ML_defaultNorm, ML_defaultNormLog, MaxML, prevEntropy->fse.matchlengthCTable, sizeof(prevEntropy->fse.matchlengthCTable), - workspace, HUF_WORKSPACE_SIZE); + workspace, wkspSize); if (ZSTD_isError(countSize)) return countSize; if (MLtype == set_compressed) lastNCount = op; @@ -2328,19 +2528,24 @@ MEM_STATIC size_t ZSTD_compressSequences_internal(seqStore_t* seqStorePtr, } } + DEBUGLOG(5, "compressed block size : %u", (unsigned)(op - ostart)); return op - ostart; } -MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - size_t srcSize, U32* workspace, int bmi2) +MEM_STATIC size_t +ZSTD_compressSequences(seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* workspace, size_t wkspSize, + int bmi2) { size_t const cSize = ZSTD_compressSequences_internal( - seqStorePtr, prevEntropy, nextEntropy, cctxParams, dst, dstCapacity, - workspace, bmi2); + seqStorePtr, prevEntropy, nextEntropy, cctxParams, + dst, dstCapacity, + workspace, wkspSize, bmi2); if (cSize == 0) return 0; /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. @@ -2362,7 +2567,7 @@ MEM_STATIC size_t ZSTD_compressSequences(seqStore_t* seqStorePtr, * assumption : strat is a valid strategy */ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMode_e dictMode) { - static const ZSTD_blockCompressor blockCompressor[3][(unsigned)ZSTD_btultra+1] = { + static const ZSTD_blockCompressor blockCompressor[3][ZSTD_STRATEGY_MAX+1] = { { ZSTD_compressBlock_fast /* default for 0 */, ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, @@ -2371,7 +2576,8 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, - ZSTD_compressBlock_btultra }, + ZSTD_compressBlock_btultra, + ZSTD_compressBlock_btultra2 }, { ZSTD_compressBlock_fast_extDict /* default for 0 */, ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, @@ -2380,6 +2586,7 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, + ZSTD_compressBlock_btultra_extDict, ZSTD_compressBlock_btultra_extDict }, { ZSTD_compressBlock_fast_dictMatchState /* default for 0 */, ZSTD_compressBlock_fast_dictMatchState, @@ -2389,14 +2596,14 @@ ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, ZSTD_dictMo ZSTD_compressBlock_lazy2_dictMatchState, ZSTD_compressBlock_btlazy2_dictMatchState, ZSTD_compressBlock_btopt_dictMatchState, + ZSTD_compressBlock_btultra_dictMatchState, ZSTD_compressBlock_btultra_dictMatchState } }; ZSTD_blockCompressor selectedCompressor; ZSTD_STATIC_ASSERT((unsigned)ZSTD_fast == 1); - assert((U32)strat >= (U32)ZSTD_fast); - assert((U32)strat <= (U32)ZSTD_btultra); - selectedCompressor = blockCompressor[(int)dictMode][(U32)strat]; + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + selectedCompressor = blockCompressor[(int)dictMode][(int)strat]; assert(selectedCompressor != NULL); return selectedCompressor; } @@ -2421,15 +2628,15 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, { ZSTD_matchState_t* const ms = &zc->blockState.matchState; size_t cSize; - DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%zu, dictLimit=%u, nextToUpdate=%u)", - dstCapacity, ms->window.dictLimit, ms->nextToUpdate); + DEBUGLOG(5, "ZSTD_compressBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + (unsigned)dstCapacity, (unsigned)ms->window.dictLimit, (unsigned)ms->nextToUpdate); assert(srcSize <= ZSTD_BLOCKSIZE_MAX); /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { - ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.searchLength); + ZSTD_ldm_skipSequences(&zc->externSeqStore, srcSize, zc->appliedParams.cParams.minMatch); cSize = 0; goto out; /* don't even attempt compression below a certain srcSize */ } @@ -2437,8 +2644,8 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, ms->opt.symbolCosts = &zc->blockState.prevCBlock->entropy; /* required for optimal parser to read stats from dictionary */ /* a gap between an attached dict and the current window is not safe, - * they must remain adjacent, and when that stops being the case, the dict - * must be unset */ + * they must remain adjacent, + * and when that stops being the case, the dict must be unset */ assert(ms->dictMatchState == NULL || ms->loadedDictEnd == ms->window.dictLimit); /* limited update after a very long match */ @@ -2495,7 +2702,9 @@ static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, dst, dstCapacity, - srcSize, zc->entropyWorkspace, zc->bmi2); + srcSize, + zc->entropyWorkspace, HUF_WORKSPACE_SIZE /* statically allocated in resetCCtx */, + zc->bmi2); out: if (!ZSTD_isError(cSize) && cSize != 0) { @@ -2535,7 +2744,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, U32 const maxDist = (U32)1 << cctx->appliedParams.cParams.windowLog; assert(cctx->appliedParams.cParams.windowLog <= 31); - DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (U32)blockSize); + DEBUGLOG(5, "ZSTD_compress_frameChunk (blockSize=%u)", (unsigned)blockSize); if (cctx->appliedParams.fParams.checksumFlag && srcSize) XXH64_update(&cctx->xxhState, src, srcSize); @@ -2583,7 +2792,7 @@ static size_t ZSTD_compress_frameChunk (ZSTD_CCtx* cctx, assert(dstCapacity >= cSize); dstCapacity -= cSize; DEBUGLOG(5, "ZSTD_compress_frameChunk: adding a block of size %u", - (U32)cSize); + (unsigned)cSize); } } if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; @@ -2606,9 +2815,9 @@ static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, size_t pos=0; assert(!(params.fParams.contentSizeFlag && pledgedSrcSize == ZSTD_CONTENTSIZE_UNKNOWN)); - if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); + if (dstCapacity < ZSTD_FRAMEHEADERSIZE_MAX) return ERROR(dstSize_tooSmall); DEBUGLOG(4, "ZSTD_writeFrameHeader : dictIDFlag : %u ; dictID : %u ; dictIDSizeCode : %u", - !params.fParams.noDictIDFlag, dictID, dictIDSizeCode); + !params.fParams.noDictIDFlag, (unsigned)dictID, (unsigned)dictIDSizeCode); if (params.format == ZSTD_f_zstd1) { MEM_writeLE32(dst, ZSTD_MAGICNUMBER); @@ -2672,7 +2881,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, size_t fhSize = 0; DEBUGLOG(5, "ZSTD_compressContinue_internal, stage: %u, srcSize: %u", - cctx->stage, (U32)srcSize); + cctx->stage, (unsigned)srcSize); if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ if (frame && (cctx->stage==ZSTDcs_init)) { @@ -2709,7 +2918,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, } } - DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (U32)cctx->blockSize); + DEBUGLOG(5, "ZSTD_compressContinue_internal (blockSize=%u)", (unsigned)cctx->blockSize); { size_t const cSize = frame ? ZSTD_compress_frameChunk (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); @@ -2721,7 +2930,7 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_UNKNOWN == (unsigned long long)-1); if (cctx->consumedSrcSize+1 > cctx->pledgedSrcSizePlusOne) { DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize >= %u", - (U32)cctx->pledgedSrcSizePlusOne-1, (U32)cctx->consumedSrcSize); + (unsigned)cctx->pledgedSrcSizePlusOne-1, (unsigned)cctx->consumedSrcSize); return ERROR(srcSize_wrong); } } @@ -2733,7 +2942,7 @@ size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (U32)srcSize); + DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } @@ -2791,6 +3000,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, case ZSTD_btlazy2: /* we want the dictionary table fully sorted */ case ZSTD_btopt: case ZSTD_btultra: + case ZSTD_btultra2: if (srcSize >= HASH_READ_SIZE) ZSTD_updateTree(ms, iend-HASH_READ_SIZE, iend); break; @@ -2861,7 +3071,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ /* fill all offset symbols to avoid garbage at end of table */ - CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable, offcodeNCount, MaxOff, offcodeLog, workspace, HUF_WORKSPACE_SIZE), + CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.offcodeCTable, + offcodeNCount, MaxOff, offcodeLog, + workspace, HUF_WORKSPACE_SIZE), dictionary_corrupted); dictPtr += offcodeHeaderSize; } @@ -2873,7 +3085,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); /* Every match length code must have non-zero probability */ CHECK_F( ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); - CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, workspace, HUF_WORKSPACE_SIZE), + CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.matchlengthCTable, + matchlengthNCount, matchlengthMaxValue, matchlengthLog, + workspace, HUF_WORKSPACE_SIZE), dictionary_corrupted); dictPtr += matchlengthHeaderSize; } @@ -2885,7 +3099,9 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); /* Every literal length code must have non-zero probability */ CHECK_F( ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); - CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, workspace, HUF_WORKSPACE_SIZE), + CHECK_E( FSE_buildCTable_wksp(bs->entropy.fse.litlengthCTable, + litlengthNCount, litlengthMaxValue, litlengthLog, + workspace, HUF_WORKSPACE_SIZE), dictionary_corrupted); dictPtr += litlengthHeaderSize; } @@ -3023,7 +3239,7 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di ZSTD_parameters const params = ZSTD_getParams(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); ZSTD_CCtx_params const cctxParams = ZSTD_assignParamsToCCtxParams(cctx->requestedParams, params); - DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (U32)dictSize); + DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); return ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } @@ -3067,7 +3283,7 @@ static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) if (cctx->appliedParams.fParams.checksumFlag) { U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); if (dstCapacity<4) return ERROR(dstSize_tooSmall); - DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", checksum); + DEBUGLOG(4, "ZSTD_writeEpilogue: write checksum : %08X", (unsigned)checksum); MEM_writeLE32(op, checksum); op += 4; } @@ -3093,7 +3309,7 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, DEBUGLOG(4, "end of frame : controlling src size"); if (cctx->pledgedSrcSizePlusOne != cctx->consumedSrcSize+1) { DEBUGLOG(4, "error : pledgedSrcSize = %u, while realSrcSize = %u", - (U32)cctx->pledgedSrcSizePlusOne-1, (U32)cctx->consumedSrcSize); + (unsigned)cctx->pledgedSrcSizePlusOne-1, (unsigned)cctx->consumedSrcSize); return ERROR(srcSize_wrong); } } return cSize + endResult; @@ -3139,7 +3355,7 @@ size_t ZSTD_compress_advanced_internal( const void* dict,size_t dictSize, ZSTD_CCtx_params params) { - DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (U32)srcSize); + DEBUGLOG(4, "ZSTD_compress_advanced_internal (srcSize:%u)", (unsigned)srcSize); CHECK_F( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, params, srcSize, ZSTDb_not_buffered) ); @@ -3163,7 +3379,7 @@ size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, const void* src, size_t srcSize, int compressionLevel) { - DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (U32)srcSize); + DEBUGLOG(4, "ZSTD_compressCCtx (srcSize=%u)", (unsigned)srcSize); assert(cctx != NULL); return ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); } @@ -3189,7 +3405,7 @@ size_t ZSTD_estimateCDictSize_advanced( size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod) { - DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (U32)sizeof(ZSTD_CDict)); + DEBUGLOG(5, "sizeof(ZSTD_CDict) : %u", (unsigned)sizeof(ZSTD_CDict)); return sizeof(ZSTD_CDict) + HUF_WORKSPACE_SIZE + ZSTD_sizeof_matchState(&cParams, /* forCCtx */ 0) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); } @@ -3203,7 +3419,7 @@ size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel) size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) { if (cdict==NULL) return 0; /* support sizeof on NULL */ - DEBUGLOG(5, "sizeof(*cdict) : %u", (U32)sizeof(*cdict)); + DEBUGLOG(5, "sizeof(*cdict) : %u", (unsigned)sizeof(*cdict)); return cdict->workspaceSize + (cdict->dictBuffer ? cdict->dictContentSize : 0) + sizeof(*cdict); } @@ -3214,7 +3430,7 @@ static size_t ZSTD_initCDict_internal( ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams) { - DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (U32)dictContentType); + DEBUGLOG(3, "ZSTD_initCDict_internal (dictContentType:%u)", (unsigned)dictContentType); assert(!ZSTD_checkCParams(cParams)); cdict->matchState.cParams = cParams; if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dictBuffer) || (!dictSize)) { @@ -3264,7 +3480,7 @@ ZSTD_CDict* ZSTD_createCDict_advanced(const void* dictBuffer, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_compressionParameters cParams, ZSTD_customMem customMem) { - DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (U32)dictContentType); + DEBUGLOG(3, "ZSTD_createCDict_advanced, mode %u", (unsigned)dictContentType); if (!customMem.customAlloc ^ !customMem.customFree) return NULL; { ZSTD_CDict* const cdict = (ZSTD_CDict*)ZSTD_malloc(sizeof(ZSTD_CDict), customMem); @@ -3345,7 +3561,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( void* ptr; if ((size_t)workspace & 7) return NULL; /* 8-aligned */ DEBUGLOG(4, "(workspaceSize < neededSize) : (%u < %u) => %u", - (U32)workspaceSize, (U32)neededSize, (U32)(workspaceSize < neededSize)); + (unsigned)workspaceSize, (unsigned)neededSize, (unsigned)(workspaceSize < neededSize)); if (workspaceSize < neededSize) return NULL; if (dictLoadMethod == ZSTD_dlm_byCopy) { @@ -3505,7 +3721,7 @@ static size_t ZSTD_resetCStream_internal(ZSTD_CStream* cctx, size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) { ZSTD_CCtx_params params = zcs->requestedParams; - DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (U32)pledgedSrcSize); + DEBUGLOG(4, "ZSTD_resetCStream: pledgedSrcSize = %u", (unsigned)pledgedSrcSize); if (pledgedSrcSize==0) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; params.fParams.contentSizeFlag = 1; return ZSTD_resetCStream_internal(zcs, NULL, 0, ZSTD_dct_auto, zcs->cdict, params, pledgedSrcSize); @@ -3525,7 +3741,7 @@ size_t ZSTD_initCStream_internal(ZSTD_CStream* zcs, assert(!((dict) && (cdict))); /* either dict or cdict, not both */ if (dict && dictSize >= 8) { - DEBUGLOG(4, "loading dictionary of size %u", (U32)dictSize); + DEBUGLOG(4, "loading dictionary of size %u", (unsigned)dictSize); if (zcs->staticSize) { /* static CCtx : never uses malloc */ /* incompatible with internal cdict creation */ return ERROR(memory_allocation); @@ -3584,7 +3800,7 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, ZSTD_parameters params, unsigned long long pledgedSrcSize) { DEBUGLOG(4, "ZSTD_initCStream_advanced: pledgedSrcSize=%u, flag=%u", - (U32)pledgedSrcSize, params.fParams.contentSizeFlag); + (unsigned)pledgedSrcSize, params.fParams.contentSizeFlag); CHECK_F( ZSTD_checkCParams(params.cParams) ); if ((pledgedSrcSize==0) && (params.fParams.contentSizeFlag==0)) pledgedSrcSize = ZSTD_CONTENTSIZE_UNKNOWN; /* for compatibility with older programs relying on this behavior. Users should now specify ZSTD_CONTENTSIZE_UNKNOWN. This line will be removed in the future. */ zcs->requestedParams = ZSTD_assignParamsToCCtxParams(zcs->requestedParams, params); @@ -3612,8 +3828,15 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) /*====== Compression ======*/ -MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) +{ + size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; +} + +static size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t const length = MIN(dstCapacity, srcSize); if (length) memcpy(dst, src, length); @@ -3621,7 +3844,7 @@ MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, } /** ZSTD_compressStream_generic(): - * internal function for all *compressStream*() variants and *compress_generic() + * internal function for all *compressStream*() variants * non-static, because can be called from zstdmt_compress.c * @return : hint size for next input */ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, @@ -3638,7 +3861,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, U32 someMoreWork = 1; /* check expectations */ - DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (U32)flushMode); + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); assert(zcs->inBuff != NULL); assert(zcs->inBuffSize > 0); assert(zcs->outBuff != NULL); @@ -3660,12 +3883,12 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* shortcut to compression pass directly into output buffer */ size_t const cSize = ZSTD_compressEnd(zcs, op, oend-op, ip, iend-ip); - DEBUGLOG(4, "ZSTD_compressEnd : %u", (U32)cSize); + DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); if (ZSTD_isError(cSize)) return cSize; ip = iend; op += cSize; zcs->frameEnded = 1; - ZSTD_CCtx_reset(zcs); + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); someMoreWork = 0; break; } /* complete loading into inBuffer */ @@ -3709,7 +3932,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (zcs->inBuffTarget > zcs->inBuffSize) zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; DEBUGLOG(5, "inBuffTarget:%u / inBuffSize:%u", - (U32)zcs->inBuffTarget, (U32)zcs->inBuffSize); + (unsigned)zcs->inBuffTarget, (unsigned)zcs->inBuffSize); if (!lastBlock) assert(zcs->inBuffTarget <= zcs->inBuffSize); zcs->inToCompress = zcs->inBuffPos; @@ -3718,7 +3941,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (zcs->frameEnded) { DEBUGLOG(5, "Frame completed directly in outBuffer"); someMoreWork = 0; - ZSTD_CCtx_reset(zcs); + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); } break; } @@ -3733,7 +3956,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); DEBUGLOG(5, "toFlush: %u into %u ==> flushed: %u", - (U32)toFlush, (U32)(oend-op), (U32)flushed); + (unsigned)toFlush, (unsigned)(oend-op), (unsigned)flushed); op += flushed; zcs->outBuffFlushedSize += flushed; if (toFlush!=flushed) { @@ -3746,7 +3969,7 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (zcs->frameEnded) { DEBUGLOG(5, "Frame completed on flush"); someMoreWork = 0; - ZSTD_CCtx_reset(zcs); + ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); break; } zcs->streamStage = zcss_load; @@ -3761,28 +3984,34 @@ size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, input->pos = ip - istart; output->pos = op - ostart; if (zcs->frameEnded) return 0; - { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; - if (hintInSize==0) hintInSize = zcs->blockSize; - return hintInSize; + return ZSTD_nextInputSizeHint(zcs); +} + +static size_t ZSTD_nextInputSizeHint_MTorST(const ZSTD_CCtx* cctx) +{ +#ifdef ZSTD_MULTITHREAD + if (cctx->appliedParams.nbWorkers >= 1) { + assert(cctx->mtctx != NULL); + return ZSTDMT_nextInputSizeHint(cctx->mtctx); } +#endif + return ZSTD_nextInputSizeHint(cctx); + } size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) { - /* check conditions */ - if (output->pos > output->size) return ERROR(GENERIC); - if (input->pos > input->size) return ERROR(GENERIC); - - return ZSTD_compressStream_generic(zcs, output, input, ZSTD_e_continue); + CHECK_F( ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue) ); + return ZSTD_nextInputSizeHint_MTorST(zcs); } -size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, - ZSTD_outBuffer* output, - ZSTD_inBuffer* input, - ZSTD_EndDirective endOp) +size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp) { - DEBUGLOG(5, "ZSTD_compress_generic, endOp=%u ", (U32)endOp); + DEBUGLOG(5, "ZSTD_compressStream2, endOp=%u ", (unsigned)endOp); /* check conditions */ if (output->pos > output->size) return ERROR(GENERIC); if (input->pos > input->size) return ERROR(GENERIC); @@ -3792,9 +4021,9 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, if (cctx->streamStage == zcss_init) { ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; - memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ - assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ - DEBUGLOG(4, "ZSTD_compress_generic : transparent init stage"); + memset(&cctx->prefixDict, 0, sizeof(cctx->prefixDict)); /* single usage */ + assert(prefixDict.dict==NULL || cctx->cdict==NULL); /* only one can be set */ + DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = input->size + 1; /* auto-fix pledgedSrcSize */ params.cParams = ZSTD_getCParamsFromCCtxParams( &cctx->requestedParams, cctx->pledgedSrcSizePlusOne-1, 0 /*dictSize*/); @@ -3807,7 +4036,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, if (params.nbWorkers > 0) { /* mt context creation */ if (cctx->mtctx == NULL) { - DEBUGLOG(4, "ZSTD_compress_generic: creating new mtctx for nbWorkers=%u", + DEBUGLOG(4, "ZSTD_compressStream2: creating new mtctx for nbWorkers=%u", params.nbWorkers); cctx->mtctx = ZSTDMT_createCCtx_advanced(params.nbWorkers, cctx->customMem); if (cctx->mtctx == NULL) return ERROR(memory_allocation); @@ -3829,6 +4058,7 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, assert(cctx->streamStage == zcss_load); assert(cctx->appliedParams.nbWorkers == 0); } } + /* end of transparent initialization stage */ /* compression stage */ #ifdef ZSTD_MULTITHREAD @@ -3840,18 +4070,18 @@ size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, { size_t const flushMin = ZSTDMT_compressStream_generic(cctx->mtctx, output, input, endOp); if ( ZSTD_isError(flushMin) || (endOp == ZSTD_e_end && flushMin == 0) ) { /* compression completed */ - ZSTD_CCtx_reset(cctx); + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); } - DEBUGLOG(5, "completed ZSTD_compress_generic delegating to ZSTDMT_compressStream_generic"); + DEBUGLOG(5, "completed ZSTD_compressStream2 delegating to ZSTDMT_compressStream_generic"); return flushMin; } } #endif CHECK_F( ZSTD_compressStream_generic(cctx, output, input, endOp) ); - DEBUGLOG(5, "completed ZSTD_compress_generic"); + DEBUGLOG(5, "completed ZSTD_compressStream2"); return cctx->outBuffContentSize - cctx->outBuffFlushedSize; /* remaining to flush */ } -size_t ZSTD_compress_generic_simpleArgs ( +size_t ZSTD_compressStream2_simpleArgs ( ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos, @@ -3859,13 +4089,33 @@ size_t ZSTD_compress_generic_simpleArgs ( { ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; ZSTD_inBuffer input = { src, srcSize, *srcPos }; - /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_compress_generic(cctx, &output, &input, endOp); + /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ + size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); *dstPos = output.pos; *srcPos = input.pos; return cErr; } +size_t ZSTD_compress2(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + { size_t oPos = 0; + size_t iPos = 0; + size_t const result = ZSTD_compressStream2_simpleArgs(cctx, + dst, dstCapacity, &oPos, + src, srcSize, &iPos, + ZSTD_e_end); + if (ZSTD_isError(result)) return result; + if (result != 0) { /* compression not completed, due to lack of output space */ + assert(oPos == dstCapacity); + return ERROR(dstSize_tooSmall); + } + assert(iPos == srcSize); /* all input is expected consumed */ + return oPos; + } +} /*====== Finalize ======*/ @@ -3874,21 +4124,21 @@ size_t ZSTD_compress_generic_simpleArgs ( size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { ZSTD_inBuffer input = { NULL, 0, 0 }; - if (output->pos > output->size) return ERROR(GENERIC); - CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_flush) ); - return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ + return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); } size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { ZSTD_inBuffer input = { NULL, 0, 0 }; - if (output->pos > output->size) return ERROR(GENERIC); - CHECK_F( ZSTD_compressStream_generic(zcs, output, &input, ZSTD_e_end) ); + size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); + CHECK_F( remainingToFlush ); + if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ + /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; size_t const checksumSize = zcs->frameEnded ? 0 : zcs->appliedParams.fParams.checksumFlag * 4; - size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize + lastBlockSize + checksumSize; - DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (U32)toFlush); + size_t const toFlush = remainingToFlush + lastBlockSize + checksumSize; + DEBUGLOG(4, "ZSTD_endStream : remaining to flush : %u", (unsigned)toFlush); return toFlush; } } @@ -3905,27 +4155,27 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV /* W, C, H, S, L, TL, strat */ { 19, 12, 13, 1, 6, 1, ZSTD_fast }, /* base for negative levels */ { 19, 13, 14, 1, 7, 0, ZSTD_fast }, /* level 1 */ - { 19, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ - { 20, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ - { 20, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ - { 20, 18, 18, 2, 5, 2, ZSTD_greedy }, /* level 5 */ - { 21, 18, 19, 2, 5, 4, ZSTD_lazy }, /* level 6 */ - { 21, 18, 19, 3, 5, 8, ZSTD_lazy2 }, /* level 7 */ + { 20, 15, 16, 1, 6, 0, ZSTD_fast }, /* level 2 */ + { 21, 16, 17, 1, 5, 1, ZSTD_dfast }, /* level 3 */ + { 21, 18, 18, 1, 5, 1, ZSTD_dfast }, /* level 4 */ + { 21, 18, 19, 2, 5, 2, ZSTD_greedy }, /* level 5 */ + { 21, 19, 19, 3, 5, 4, ZSTD_greedy }, /* level 6 */ + { 21, 19, 19, 3, 5, 8, ZSTD_lazy }, /* level 7 */ { 21, 19, 19, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ { 21, 19, 20, 4, 5, 16, ZSTD_lazy2 }, /* level 9 */ - { 21, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ - { 21, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ - { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ - { 22, 21, 22, 4, 5, 32, ZSTD_btlazy2 }, /* level 13 */ - { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ - { 22, 22, 22, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ - { 22, 21, 22, 4, 5, 48, ZSTD_btopt }, /* level 16 */ - { 23, 22, 22, 4, 4, 64, ZSTD_btopt }, /* level 17 */ - { 23, 23, 22, 6, 3,256, ZSTD_btopt }, /* level 18 */ - { 23, 24, 22, 7, 3,256, ZSTD_btultra }, /* level 19 */ - { 25, 25, 23, 7, 3,256, ZSTD_btultra }, /* level 20 */ - { 26, 26, 24, 7, 3,512, ZSTD_btultra }, /* level 21 */ - { 27, 27, 25, 9, 3,999, ZSTD_btultra }, /* level 22 */ + { 22, 20, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 21, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 32, ZSTD_btlazy2 }, /* level 13 */ + { 22, 22, 23, 5, 5, 32, ZSTD_btlazy2 }, /* level 14 */ + { 22, 23, 23, 6, 5, 32, ZSTD_btlazy2 }, /* level 15 */ + { 22, 22, 22, 5, 5, 48, ZSTD_btopt }, /* level 16 */ + { 23, 23, 22, 5, 4, 64, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 3, 64, ZSTD_btultra }, /* level 18 */ + { 23, 24, 22, 7, 3,256, ZSTD_btultra2}, /* level 19 */ + { 25, 25, 23, 7, 3,256, ZSTD_btultra2}, /* level 20 */ + { 26, 26, 24, 7, 3,512, ZSTD_btultra2}, /* level 21 */ + { 27, 27, 25, 9, 3,999, ZSTD_btultra2}, /* level 22 */ }, { /* for srcSize <= 256 KB */ /* W, C, H, S, L, T, strat */ @@ -3940,18 +4190,18 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 18, 18, 19, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 18, 18, 19, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ { 18, 18, 19, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 18, 18, 19, 5, 4, 16, ZSTD_btlazy2 }, /* level 11.*/ - { 18, 19, 19, 6, 4, 16, ZSTD_btlazy2 }, /* level 12.*/ - { 18, 19, 19, 8, 4, 16, ZSTD_btlazy2 }, /* level 13 */ - { 18, 18, 19, 4, 4, 24, ZSTD_btopt }, /* level 14.*/ - { 18, 18, 19, 4, 3, 24, ZSTD_btopt }, /* level 15.*/ - { 18, 19, 19, 6, 3, 64, ZSTD_btopt }, /* level 16.*/ - { 18, 19, 19, 8, 3,128, ZSTD_btopt }, /* level 17.*/ - { 18, 19, 19, 10, 3,256, ZSTD_btopt }, /* level 18.*/ - { 18, 19, 19, 10, 3,256, ZSTD_btultra }, /* level 19.*/ - { 18, 19, 19, 11, 3,512, ZSTD_btultra }, /* level 20.*/ - { 18, 19, 19, 12, 3,512, ZSTD_btultra }, /* level 21.*/ - { 18, 19, 19, 13, 3,999, ZSTD_btultra }, /* level 22.*/ + { 18, 18, 19, 5, 4, 12, ZSTD_btlazy2 }, /* level 11.*/ + { 18, 19, 19, 7, 4, 12, ZSTD_btlazy2 }, /* level 12.*/ + { 18, 18, 19, 4, 4, 16, ZSTD_btopt }, /* level 13 */ + { 18, 18, 19, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 19, 6, 3,128, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 18, 19, 19, 6, 3,128, ZSTD_btultra2}, /* level 18.*/ + { 18, 19, 19, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 18, 19, 19, 10, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 18, 19, 19, 12, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 18, 19, 19, 13, 3,999, ZSTD_btultra2}, /* level 22.*/ }, { /* for srcSize <= 128 KB */ /* W, C, H, S, L, T, strat */ @@ -3966,26 +4216,26 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ - { 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ - { 17, 18, 17, 6, 4, 16, ZSTD_btlazy2 }, /* level 12 */ - { 17, 18, 17, 8, 4, 16, ZSTD_btlazy2 }, /* level 13.*/ - { 17, 18, 17, 4, 4, 32, ZSTD_btopt }, /* level 14.*/ - { 17, 18, 17, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ - { 17, 18, 17, 7, 3,128, ZSTD_btopt }, /* level 16.*/ - { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 17.*/ - { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 18.*/ - { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 19.*/ - { 17, 18, 17, 9, 3,256, ZSTD_btultra }, /* level 20.*/ - { 17, 18, 17, 10, 3,256, ZSTD_btultra }, /* level 21.*/ - { 17, 18, 17, 11, 3,512, ZSTD_btultra }, /* level 22.*/ + { 17, 17, 17, 5, 4, 8, ZSTD_btlazy2 }, /* level 11 */ + { 17, 18, 17, 7, 4, 12, ZSTD_btlazy2 }, /* level 12 */ + { 17, 18, 17, 3, 4, 12, ZSTD_btopt }, /* level 13.*/ + { 17, 18, 17, 4, 3, 32, ZSTD_btopt }, /* level 14.*/ + { 17, 18, 17, 6, 3,256, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 6, 3,128, ZSTD_btultra }, /* level 16.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btultra }, /* level 17.*/ + { 17, 18, 17, 10, 3,512, ZSTD_btultra }, /* level 18.*/ + { 17, 18, 17, 5, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 17, 18, 17, 7, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 17, 18, 17, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 17, 18, 17, 11, 3,999, ZSTD_btultra2}, /* level 22.*/ }, { /* for srcSize <= 16 KB */ /* W, C, H, S, L, T, strat */ { 14, 12, 13, 1, 5, 1, ZSTD_fast }, /* base for negative levels */ { 14, 14, 15, 1, 5, 0, ZSTD_fast }, /* level 1 */ { 14, 14, 15, 1, 4, 0, ZSTD_fast }, /* level 2 */ - { 14, 14, 14, 2, 4, 1, ZSTD_dfast }, /* level 3.*/ - { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 15, 2, 4, 1, ZSTD_dfast }, /* level 3 */ + { 14, 14, 14, 4, 4, 2, ZSTD_greedy }, /* level 4 */ { 14, 14, 14, 3, 4, 4, ZSTD_lazy }, /* level 5.*/ { 14, 14, 14, 4, 4, 8, ZSTD_lazy2 }, /* level 6 */ { 14, 14, 14, 6, 4, 8, ZSTD_lazy2 }, /* level 7 */ @@ -3993,17 +4243,17 @@ static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEV { 14, 15, 14, 5, 4, 8, ZSTD_btlazy2 }, /* level 9.*/ { 14, 15, 14, 9, 4, 8, ZSTD_btlazy2 }, /* level 10.*/ { 14, 15, 14, 3, 4, 12, ZSTD_btopt }, /* level 11.*/ - { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ - { 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ - { 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ - { 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ - { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ - { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ - { 14, 15, 15, 8, 3,256, ZSTD_btopt }, /* level 18.*/ - { 14, 15, 15, 6, 3,256, ZSTD_btultra }, /* level 19.*/ - { 14, 15, 15, 8, 3,256, ZSTD_btultra }, /* level 20.*/ - { 14, 15, 15, 9, 3,256, ZSTD_btultra }, /* level 21.*/ - { 14, 15, 15, 10, 3,512, ZSTD_btultra }, /* level 22.*/ + { 14, 15, 14, 4, 3, 24, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 5, 3, 32, ZSTD_btultra }, /* level 13.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btultra }, /* level 14.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra }, /* level 15.*/ + { 14, 15, 15, 5, 3, 48, ZSTD_btultra2}, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btultra2}, /* level 17.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btultra2}, /* level 18.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btultra2}, /* level 19.*/ + { 14, 15, 15, 8, 3,512, ZSTD_btultra2}, /* level 20.*/ + { 14, 15, 15, 9, 3,512, ZSTD_btultra2}, /* level 21.*/ + { 14, 15, 15, 10, 3,999, ZSTD_btultra2}, /* level 22.*/ }, }; @@ -4022,8 +4272,8 @@ ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long l if (compressionLevel > ZSTD_MAX_CLEVEL) row = ZSTD_MAX_CLEVEL; { ZSTD_compressionParameters cp = ZSTD_defaultCParameters[tableID][row]; if (compressionLevel < 0) cp.targetLength = (unsigned)(-compressionLevel); /* acceleration factor */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); } - + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize); + } } /*! ZSTD_getParams() : diff --git a/C/zstd/zstd_compress_internal.h b/C/zstd/zstd_compress_internal.h index 43f7c148..29bca598 100644 --- a/C/zstd/zstd_compress_internal.h +++ b/C/zstd/zstd_compress_internal.h @@ -48,12 +48,6 @@ extern "C" { typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; typedef enum { zcss_init=0, zcss_load, zcss_flush } ZSTD_cStreamStage; -typedef enum { - ZSTD_dictDefaultAttach = 0, - ZSTD_dictForceAttach = 1, - ZSTD_dictForceCopy = -1, -} ZSTD_dictAttachPref_e; - typedef struct ZSTD_prefixDict_s { const void* dict; size_t dictSize; @@ -96,10 +90,10 @@ typedef enum { zop_dynamic=0, zop_predef } ZSTD_OptPrice_e; typedef struct { /* All tables are allocated inside cctx->workspace by ZSTD_resetCCtx_internal() */ - U32* litFreq; /* table of literals statistics, of size 256 */ - U32* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ - U32* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ - U32* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ + unsigned* litFreq; /* table of literals statistics, of size 256 */ + unsigned* litLengthFreq; /* table of litLength statistics, of size (MaxLL+1) */ + unsigned* matchLengthFreq; /* table of matchLength statistics, of size (MaxML+1) */ + unsigned* offCodeFreq; /* table of offCode statistics, of size (MaxOff+1) */ ZSTD_match_t* matchTable; /* list of found matches, of size ZSTD_OPT_NUM+1 */ ZSTD_optimal_t* priceTable; /* All positions tracked by optimal parser, of size ZSTD_OPT_NUM+1 */ @@ -139,7 +133,7 @@ struct ZSTD_matchState_t { U32* hashTable3; U32* chainTable; optState_t opt; /* optimal parser state */ - const ZSTD_matchState_t *dictMatchState; + const ZSTD_matchState_t * dictMatchState; ZSTD_compressionParameters cParams; }; @@ -167,7 +161,7 @@ typedef struct { U32 hashLog; /* Log size of hashTable */ U32 bucketSizeLog; /* Log bucket size for collision resolution, at most 8 */ U32 minMatchLength; /* Minimum match length */ - U32 hashEveryLog; /* Log number of entries to skip */ + U32 hashRateLog; /* Log number of entries to skip */ U32 windowLog; /* Window log for the LDM */ } ldmParams_t; @@ -196,9 +190,10 @@ struct ZSTD_CCtx_params_s { ZSTD_dictAttachPref_e attachDictPref; /* Multithreading: used to pass parameters to mtctx */ - unsigned nbWorkers; - unsigned jobSize; - unsigned overlapSizeLog; + int nbWorkers; + size_t jobSize; + int overlapLog; + int rsyncable; /* Long distance matching parameters */ ldmParams_t ldmParams; @@ -498,6 +493,64 @@ MEM_STATIC size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) } } +/** ZSTD_ipow() : + * Return base^exponent. + */ +static U64 ZSTD_ipow(U64 base, U64 exponent) +{ + U64 power = 1; + while (exponent) { + if (exponent & 1) power *= base; + exponent >>= 1; + base *= base; + } + return power; +} + +#define ZSTD_ROLL_HASH_CHAR_OFFSET 10 + +/** ZSTD_rollingHash_append() : + * Add the buffer to the hash value. + */ +static U64 ZSTD_rollingHash_append(U64 hash, void const* buf, size_t size) +{ + BYTE const* istart = (BYTE const*)buf; + size_t pos; + for (pos = 0; pos < size; ++pos) { + hash *= prime8bytes; + hash += istart[pos] + ZSTD_ROLL_HASH_CHAR_OFFSET; + } + return hash; +} + +/** ZSTD_rollingHash_compute() : + * Compute the rolling hash value of the buffer. + */ +MEM_STATIC U64 ZSTD_rollingHash_compute(void const* buf, size_t size) +{ + return ZSTD_rollingHash_append(0, buf, size); +} + +/** ZSTD_rollingHash_primePower() : + * Compute the primePower to be passed to ZSTD_rollingHash_rotate() for a hash + * over a window of length bytes. + */ +MEM_STATIC U64 ZSTD_rollingHash_primePower(U32 length) +{ + return ZSTD_ipow(prime8bytes, length - 1); +} + +/** ZSTD_rollingHash_rotate() : + * Rotate the rolling hash by one byte. + */ +MEM_STATIC U64 ZSTD_rollingHash_rotate(U64 hash, BYTE toRemove, BYTE toAdd, U64 primePower) +{ + hash -= (toRemove + ZSTD_ROLL_HASH_CHAR_OFFSET) * primePower; + hash *= prime8bytes; + hash += toAdd + ZSTD_ROLL_HASH_CHAR_OFFSET; + return hash; +} + /*-************************************* * Round buffer management ***************************************/ @@ -626,20 +679,23 @@ MEM_STATIC U32 ZSTD_window_correctOverflow(ZSTD_window_t* window, U32 cycleLog, * dictMatchState mode, lowLimit and dictLimit are the same, and the dictionary * is below them. forceWindow and dictMatchState are therefore incompatible. */ -MEM_STATIC void ZSTD_window_enforceMaxDist(ZSTD_window_t* window, - void const* srcEnd, U32 maxDist, - U32* loadedDictEndPtr, - const ZSTD_matchState_t** dictMatchStatePtr) +MEM_STATIC void +ZSTD_window_enforceMaxDist(ZSTD_window_t* window, + void const* srcEnd, + U32 maxDist, + U32* loadedDictEndPtr, + const ZSTD_matchState_t** dictMatchStatePtr) { - U32 const current = (U32)((BYTE const*)srcEnd - window->base); - U32 loadedDictEnd = loadedDictEndPtr != NULL ? *loadedDictEndPtr : 0; - DEBUGLOG(5, "ZSTD_window_enforceMaxDist: current=%u, maxDist=%u", current, maxDist); - if (current > maxDist + loadedDictEnd) { - U32 const newLowLimit = current - maxDist; + U32 const blockEndIdx = (U32)((BYTE const*)srcEnd - window->base); + U32 loadedDictEnd = (loadedDictEndPtr != NULL) ? *loadedDictEndPtr : 0; + DEBUGLOG(5, "ZSTD_window_enforceMaxDist: blockEndIdx=%u, maxDist=%u", + (unsigned)blockEndIdx, (unsigned)maxDist); + if (blockEndIdx > maxDist + loadedDictEnd) { + U32 const newLowLimit = blockEndIdx - maxDist; if (window->lowLimit < newLowLimit) window->lowLimit = newLowLimit; if (window->dictLimit < window->lowLimit) { DEBUGLOG(5, "Update dictLimit to match lowLimit, from %u to %u", - window->dictLimit, window->lowLimit); + (unsigned)window->dictLimit, (unsigned)window->lowLimit); window->dictLimit = window->lowLimit; } if (loadedDictEndPtr) @@ -690,20 +746,23 @@ MEM_STATIC U32 ZSTD_window_update(ZSTD_window_t* window, /* debug functions */ +#if (DEBUGLEVEL>=2) MEM_STATIC double ZSTD_fWeight(U32 rawStat) { U32 const fp_accuracy = 8; U32 const fp_multiplier = (1 << fp_accuracy); - U32 const stat = rawStat + 1; - U32 const hb = ZSTD_highbit32(stat); + U32 const newStat = rawStat + 1; + U32 const hb = ZSTD_highbit32(newStat); U32 const BWeight = hb * fp_multiplier; - U32 const FWeight = (stat << fp_accuracy) >> hb; + U32 const FWeight = (newStat << fp_accuracy) >> hb; U32 const weight = BWeight + FWeight; assert(hb + fp_accuracy < 31); return (double)weight / fp_multiplier; } +/* display a table content, + * listing each element, its frequency, and its predicted bit cost */ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) { unsigned u, sum; @@ -715,6 +774,9 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) } } +#endif + + #if defined (__cplusplus) } #endif diff --git a/C/zstd/zstd_ddict.c b/C/zstd/zstd_ddict.c new file mode 100644 index 00000000..2ad04406 --- /dev/null +++ b/C/zstd/zstd_ddict.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include /* memcpy, memmove, memset */ +#include "cpu.h" /* bmi2 */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + CHECK_E( ZSTD_loadDEntropy(&ddict->entropy, + ddict->dictContent, ddict->dictSize), + dictionary_corrupted ); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + CHECK_F( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) ); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if (!customMem.customAlloc ^ !customMem.customFree) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_free(ddict->dictBuffer, cMem); + ZSTD_free(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); +} diff --git a/C/zstd/zstd_ddict.h b/C/zstd/zstd_ddict.h new file mode 100644 index 00000000..0479d11b --- /dev/null +++ b/C/zstd/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include /* size_t */ +#include "zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/C/zstd/zstd_decompress.c b/C/zstd/zstd_decompress.c index 711b5b6d..feef1ef6 100644 --- a/C/zstd/zstd_decompress.c +++ b/C/zstd/zstd_decompress.c @@ -37,12 +37,12 @@ * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). */ #ifndef ZSTD_MAXWINDOWSIZE_DEFAULT -# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_DEFAULTMAX) + 1) +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) #endif /*! * NO_FORWARD_PROGRESS_MAX : - * maximum allowed nb of calls to ZSTD_decompressStream() and ZSTD_decompress_generic() + * maximum allowed nb of calls to ZSTD_decompressStream() * without any forward progress * (defined as: no byte read from input, and no byte flushed to output) * before triggering an error. @@ -56,128 +56,25 @@ * Dependencies *********************************************************/ #include /* memcpy, memmove, memset */ -#include "compiler.h" /* prefetch */ #include "cpu.h" /* bmi2 */ #include "mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" #define HUF_STATIC_LINKING_ONLY #include "huf.h" -#include "zstd_internal.h" +#include "zstd_internal.h" /* blockProperties_t */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) # include "zstd_legacy.h" #endif -static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict); -static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict); - - -/*-************************************* -* Errors -***************************************/ -#define ZSTD_isError ERR_isError /* for inlining */ -#define FSE_isError ERR_isError -#define HUF_isError ERR_isError - - -/*_******************************************************* -* Memory operations -**********************************************************/ -static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } - /*-************************************************************* * Context management ***************************************************************/ -typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, - ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, - ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, - ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; - -typedef enum { zdss_init=0, zdss_loadHeader, - zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; - - -typedef struct { - U32 fastMode; - U32 tableLog; -} ZSTD_seqSymbol_header; - -typedef struct { - U16 nextState; - BYTE nbAdditionalBits; - BYTE nbBits; - U32 baseValue; -} ZSTD_seqSymbol; - -#define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) - -typedef struct { - ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ - ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ - ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ - U32 rep[ZSTD_REP_NUM]; -} ZSTD_entropyDTables_t; - -struct ZSTD_DCtx_s -{ - const ZSTD_seqSymbol* LLTptr; - const ZSTD_seqSymbol* MLTptr; - const ZSTD_seqSymbol* OFTptr; - const HUF_DTable* HUFptr; - ZSTD_entropyDTables_t entropy; - U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ - const void* previousDstEnd; /* detect continuity */ - const void* prefixStart; /* start of current segment */ - const void* virtualStart; /* virtual start of previous segment if it was just before current one */ - const void* dictEnd; /* end of previous segment */ - size_t expected; - ZSTD_frameHeader fParams; - U64 decodedSize; - blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ - ZSTD_dStage stage; - U32 litEntropy; - U32 fseEntropy; - XXH64_state_t xxhState; - size_t headerSize; - ZSTD_format_e format; - const BYTE* litPtr; - ZSTD_customMem customMem; - size_t litSize; - size_t rleSize; - size_t staticSize; - int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ - - /* dictionary */ - ZSTD_DDict* ddictLocal; - const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ - U32 dictID; - int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ - - /* streaming */ - ZSTD_dStreamStage streamStage; - char* inBuff; - size_t inBuffSize; - size_t inPos; - size_t maxWindowSize; - char* outBuff; - size_t outBuffSize; - size_t outStart; - size_t outEnd; - size_t lhSize; - void* legacyContext; - U32 previousLegacyVersion; - U32 legacyVersion; - U32 hostageByte; - int noForwardProgress; - - /* workspace */ - BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; - BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; -}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ - size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { if (dctx==NULL) return 0; /* support sizeof NULL */ @@ -192,8 +89,8 @@ size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } static size_t ZSTD_startingInputLength(ZSTD_format_e format) { size_t const startingInputLength = (format==ZSTD_f_zstd1_magicless) ? - ZSTD_frameHeaderSize_prefix - ZSTD_FRAMEIDSIZE : - ZSTD_frameHeaderSize_prefix; + ZSTD_FRAMEHEADERSIZE_PREFIX - ZSTD_FRAMEIDSIZE : + ZSTD_FRAMEHEADERSIZE_PREFIX; ZSTD_STATIC_ASSERT(ZSTD_FRAMEHEADERSIZE_PREFIX >= ZSTD_FRAMEIDSIZE); /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); @@ -290,7 +187,7 @@ unsigned ZSTD_isFrame(const void* buffer, size_t size) if (size < ZSTD_FRAMEIDSIZE) return 0; { U32 const magic = MEM_readLE32(buffer); if (magic == ZSTD_MAGICNUMBER) return 1; - if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; } #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(buffer, size)) return 1; @@ -345,10 +242,10 @@ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, s if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ - if (srcSize < ZSTD_skippableHeaderSize) - return ZSTD_skippableHeaderSize; /* magic number + frame length */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ memset(zfhPtr, 0, sizeof(*zfhPtr)); zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); zfhPtr->frameType = ZSTD_skippableFrame; @@ -446,6 +343,21 @@ unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) } } } +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ERROR(srcSize_wrong); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + if ((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32) + return ERROR(frameParameter_unsupported); + + return skippableHeaderSize + sizeU32; +} + /** ZSTD_findDecompressedSize() : * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or @@ -455,15 +367,13 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; - while (srcSize >= ZSTD_frameHeaderSize_prefix) { + while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { U32 const magicNumber = MEM_readLE32(src); - if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - size_t skippableSize; - if (srcSize < ZSTD_skippableHeaderSize) - return ERROR(srcSize_wrong); - skippableSize = MEM_readLE32((const BYTE *)src + ZSTD_FRAMEIDSIZE) - + ZSTD_skippableHeaderSize; + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) + return skippableSize; if (srcSize < skippableSize) { return ZSTD_CONTENTSIZE_ERROR; } @@ -496,9 +406,9 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) } /** ZSTD_getDecompressedSize() : -* compatible with legacy mode -* @return : decompressed size if known, 0 otherwise - note : 0 can mean any of the following : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : - frame content is empty - decompressed size field is not present in frame header - frame header unknown / not supported @@ -512,8 +422,8 @@ unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) /** ZSTD_decodeFrameHeader() : -* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). -* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) { size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); @@ -526,1275 +436,6 @@ static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t he } -/*-************************************************************* - * Block decoding - ***************************************************************/ - -/*! ZSTD_getcBlockSize() : -* Provides the size of compressed block from block header `src` */ -size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, - blockProperties_t* bpPtr) -{ - if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); - { U32 const cBlockHeader = MEM_readLE24(src); - U32 const cSize = cBlockHeader >> 3; - bpPtr->lastBlock = cBlockHeader & 1; - bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); - bpPtr->origSize = cSize; /* only useful for RLE */ - if (bpPtr->blockType == bt_rle) return 1; - if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); - return cSize; - } -} - - -static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - if (dst==NULL) return ERROR(dstSize_tooSmall); - if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); - memcpy(dst, src, srcSize); - return srcSize; -} - - -static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - size_t regenSize) -{ - if (srcSize != 1) return ERROR(srcSize_wrong); - if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); - memset(dst, *(const BYTE*)src, regenSize); - return regenSize; -} - -/* Hidden declaration for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize); -/*! ZSTD_decodeLiteralsBlock() : - * @return : nb of bytes read from src (< srcSize ) - * note : symbol not declared but exposed for fullbench */ -size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, - const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ -{ - if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); - - { const BYTE* const istart = (const BYTE*) src; - symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); - - switch(litEncType) - { - case set_repeat: - if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); - /* fall-through */ - - case set_compressed: - if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ - { size_t lhSize, litSize, litCSize; - U32 singleStream=0; - U32 const lhlCode = (istart[0] >> 2) & 3; - U32 const lhc = MEM_readLE32(istart); - switch(lhlCode) - { - case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ - /* 2 - 2 - 10 - 10 */ - singleStream = !lhlCode; - lhSize = 3; - litSize = (lhc >> 4) & 0x3FF; - litCSize = (lhc >> 14) & 0x3FF; - break; - case 2: - /* 2 - 2 - 14 - 14 */ - lhSize = 4; - litSize = (lhc >> 4) & 0x3FFF; - litCSize = lhc >> 18; - break; - case 3: - /* 2 - 2 - 18 - 18 */ - lhSize = 5; - litSize = (lhc >> 4) & 0x3FFFF; - litCSize = (lhc >> 22) + (istart[4] << 10); - break; - } - if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); - if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); - - /* prefetch huffman table if cold */ - if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { - PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); - } - - if (HUF_isError((litEncType==set_repeat) ? - ( singleStream ? - HUF_decompress1X_usingDTable_bmi2(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr, dctx->bmi2) : - HUF_decompress4X_usingDTable_bmi2(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr, dctx->bmi2) ) : - ( singleStream ? - HUF_decompress1X1_DCtx_wksp_bmi2(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->workspace, sizeof(dctx->workspace), dctx->bmi2) : - HUF_decompress4X_hufOnly_wksp_bmi2(dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->workspace, sizeof(dctx->workspace), dctx->bmi2)))) - return ERROR(corruption_detected); - - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - dctx->litEntropy = 1; - if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); - return litCSize + lhSize; - } - - case set_basic: - { size_t litSize, lhSize; - U32 const lhlCode = ((istart[0]) >> 2) & 3; - switch(lhlCode) - { - case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = MEM_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = MEM_readLE24(istart) >> 4; - break; - } - - if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ - if (litSize+lhSize > srcSize) return ERROR(corruption_detected); - memcpy(dctx->litBuffer, istart+lhSize, litSize); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); - return lhSize+litSize; - } - /* direct reference into compressed stream */ - dctx->litPtr = istart+lhSize; - dctx->litSize = litSize; - return lhSize+litSize; - } - - case set_rle: - { U32 const lhlCode = ((istart[0]) >> 2) & 3; - size_t litSize, lhSize; - switch(lhlCode) - { - case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ - lhSize = 1; - litSize = istart[0] >> 3; - break; - case 1: - lhSize = 2; - litSize = MEM_readLE16(istart) >> 4; - break; - case 3: - lhSize = 3; - litSize = MEM_readLE24(istart) >> 4; - if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ - break; - } - if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); - memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); - dctx->litPtr = dctx->litBuffer; - dctx->litSize = litSize; - return lhSize+1; - } - default: - return ERROR(corruption_detected); /* impossible */ - } - } -} - -/* Default FSE distribution tables. - * These are pre-calculated FSE decoding tables using default distributions as defined in specification : - * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions - * They were generated programmatically with following method : - * - start from default distributions, present in /lib/common/zstd_internal.h - * - generate tables normally, using ZSTD_buildFSETable() - * - printout the content of tables - * - pretify output, report below, test with fuzzer to ensure it's correct */ - -/* Default FSE distribution table for Literal Lengths */ -static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; - DTableH->fastMode = 0; - - cell->nbBits = 0; - cell->nextState = 0; - assert(nbAddBits < 255); - cell->nbAdditionalBits = (BYTE)nbAddBits; - cell->baseValue = baseValue; -} - - -/* ZSTD_buildFSETable() : - * generate FSE decoding table for one symbol (ll, ml or off) */ -static void -ZSTD_buildFSETable(ZSTD_seqSymbol* dt, - const short* normalizedCounter, unsigned maxSymbolValue, - const U32* baseValue, const U32* nbAdditionalBits, - unsigned tableLog) -{ - ZSTD_seqSymbol* const tableDecode = dt+1; - U16 symbolNext[MaxSeq+1]; - - U32 const maxSV1 = maxSymbolValue + 1; - U32 const tableSize = 1 << tableLog; - U32 highThreshold = tableSize-1; - - /* Sanity Checks */ - assert(maxSymbolValue <= MaxSeq); - assert(tableLog <= MaxFSELog); - - /* Init, lay down lowprob symbols */ - { ZSTD_seqSymbol_header DTableH; - DTableH.tableLog = tableLog; - DTableH.fastMode = 1; - { S16 const largeLimit= (S16)(1 << (tableLog-1)); - U32 s; - for (s=0; s= largeLimit) DTableH.fastMode=0; - symbolNext[s] = normalizedCounter[s]; - } } } - memcpy(dt, &DTableH, sizeof(DTableH)); - } - - /* Spread symbols */ - { U32 const tableMask = tableSize-1; - U32 const step = FSE_TABLESTEP(tableSize); - U32 s, position = 0; - for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ - } } - assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - - /* Build Decoding table */ - { U32 u; - for (u=0; u max) return ERROR(corruption_detected); - { U32 const symbol = *(const BYTE*)src; - U32 const baseline = baseValue[symbol]; - U32 const nbBits = nbAdditionalBits[symbol]; - ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); - } - *DTablePtr = DTableSpace; - return 1; - case set_basic : - *DTablePtr = defaultTable; - return 0; - case set_repeat: - if (!flagRepeatTable) return ERROR(corruption_detected); - /* prefetch FSE table if used */ - if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { - const void* const pStart = *DTablePtr; - size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); - PREFETCH_AREA(pStart, pSize); - } - return 0; - case set_compressed : - { U32 tableLog; - S16 norm[MaxSeq+1]; - size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); - if (FSE_isError(headerSize)) return ERROR(corruption_detected); - if (tableLog > maxLog) return ERROR(corruption_detected); - ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); - *DTablePtr = DTableSpace; - return headerSize; - } - default : /* impossible */ - assert(0); - return ERROR(GENERIC); - } -} - -static const U32 LL_base[MaxLL+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 18, 20, 22, 24, 28, 32, 40, - 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, - 0x2000, 0x4000, 0x8000, 0x10000 }; - -static const U32 OF_base[MaxOff+1] = { - 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, - 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, - 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, - 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; - -static const U32 OF_bits[MaxOff+1] = { - 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, - 24, 25, 26, 27, 28, 29, 30, 31 }; - -static const U32 ML_base[MaxML+1] = { - 3, 4, 5, 6, 7, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 37, 39, 41, 43, 47, 51, 59, - 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, - 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; - -/* Hidden delcaration for fullbench */ -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, - const void* src, size_t srcSize); - -size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, - const void* src, size_t srcSize) -{ - const BYTE* const istart = (const BYTE* const)src; - const BYTE* const iend = istart + srcSize; - const BYTE* ip = istart; - int nbSeq; - DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); - - /* check */ - if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); - - /* SeqHead */ - nbSeq = *ip++; - if (!nbSeq) { *nbSeqPtr=0; return 1; } - if (nbSeq > 0x7F) { - if (nbSeq == 0xFF) { - if (ip+2 > iend) return ERROR(srcSize_wrong); - nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; - } else { - if (ip >= iend) return ERROR(srcSize_wrong); - nbSeq = ((nbSeq-0x80)<<8) + *ip++; - } - } - *nbSeqPtr = nbSeq; - - /* FSE table descriptors */ - if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ - { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); - symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); - symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); - ip++; - - /* Build DTables */ - { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, - LLtype, MaxLL, LLFSELog, - ip, iend-ip, - LL_base, LL_bits, - LL_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq); - if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); - ip += llhSize; - } - - { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, - OFtype, MaxOff, OffFSELog, - ip, iend-ip, - OF_base, OF_bits, - OF_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq); - if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); - ip += ofhSize; - } - - { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, - MLtype, MaxML, MLFSELog, - ip, iend-ip, - ML_base, ML_bits, - ML_defaultDTable, dctx->fseEntropy, - dctx->ddictIsCold, nbSeq); - if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); - ip += mlhSize; - } - } - - /* prefetch dictionary content */ - if (dctx->ddictIsCold) { - size_t const dictSize = (const char*)dctx->prefixStart - (const char*)dctx->virtualStart; - size_t const psmin = MIN(dictSize, (size_t)(64*nbSeq) /* heuristic */ ); - size_t const pSize = MIN(psmin, 128 KB /* protection */ ); - const void* const pStart = (const char*)dctx->dictEnd - pSize; - PREFETCH_AREA(pStart, pSize); - dctx->ddictIsCold = 0; - } - - return ip-istart; -} - - -typedef struct { - size_t litLength; - size_t matchLength; - size_t offset; - const BYTE* match; -} seq_t; - -typedef struct { - size_t state; - const ZSTD_seqSymbol* table; -} ZSTD_fseState; - -typedef struct { - BIT_DStream_t DStream; - ZSTD_fseState stateLL; - ZSTD_fseState stateOffb; - ZSTD_fseState stateML; - size_t prevOffset[ZSTD_REP_NUM]; - const BYTE* prefixStart; - const BYTE* dictEnd; - size_t pos; -} seqState_t; - - -FORCE_NOINLINE -size_t ZSTD_execSequenceLast7(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - /* check */ - if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ - - /* copy literals */ - if (op < oend_w) { - ZSTD_wildcopy(op, *litPtr, oend_w - op); - *litPtr += oend_w - op; - op = oend_w; - } - while (op < oLitEnd) *op++ = *(*litPtr)++; - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - base)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); - match = dictEnd - (base-match); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = base; - } } - while (op < oMatchEnd) *op++ = *match++; - return sequenceLength; -} - - -HINT_INLINE -size_t ZSTD_execSequence(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = oLitEnd - sequence.offset; - - /* check */ - if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); - - /* copy Literals */ - ZSTD_copy8(op, *litPtr); - if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix -> go into extDict */ - if (sequence.offset > (size_t)(oLitEnd - virtualStart)) - return ERROR(corruption_detected); - match = dictEnd + (match - prefixStart); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } - } } - /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - - -HINT_INLINE -size_t ZSTD_execSequenceLong(BYTE* op, - BYTE* const oend, seq_t sequence, - const BYTE** litPtr, const BYTE* const litLimit, - const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd) -{ - BYTE* const oLitEnd = op + sequence.litLength; - size_t const sequenceLength = sequence.litLength + sequence.matchLength; - BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ - BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; - const BYTE* const iLitEnd = *litPtr + sequence.litLength; - const BYTE* match = sequence.match; - - /* check */ - if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ - if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); - - /* copy Literals */ - ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ - if (sequence.litLength > 8) - ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ - op = oLitEnd; - *litPtr = iLitEnd; /* update for next sequence */ - - /* copy Match */ - if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { - /* offset beyond prefix */ - if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected); - if (match + sequence.matchLength <= dictEnd) { - memmove(oLitEnd, match, sequence.matchLength); - return sequenceLength; - } - /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; - memmove(oLitEnd, match, length1); - op = oLitEnd + length1; - sequence.matchLength -= length1; - match = prefixStart; - if (op > oend_w || sequence.matchLength < MINMATCH) { - U32 i; - for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; - return sequenceLength; - } - } } - assert(op <= oend_w); - assert(sequence.matchLength >= MINMATCH); - - /* match within prefix */ - if (sequence.offset < 8) { - /* close range match, overlap */ - static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ - static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ - int const sub2 = dec64table[sequence.offset]; - op[0] = match[0]; - op[1] = match[1]; - op[2] = match[2]; - op[3] = match[3]; - match += dec32table[sequence.offset]; - ZSTD_copy4(op+4, match); - match -= sub2; - } else { - ZSTD_copy8(op, match); - } - op += 8; match += 8; - - if (oMatchEnd > oend-(16-MINMATCH)) { - if (op < oend_w) { - ZSTD_wildcopy(op, match, oend_w - op); - match += oend_w - op; - op = oend_w; - } - while (op < oMatchEnd) *op++ = *match++; - } else { - ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ - } - return sequenceLength; -} - -static void -ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) -{ - const void* ptr = dt; - const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; - DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); - DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", - (U32)DStatePtr->state, DTableH->tableLog); - BIT_reloadDStream(bitD); - DStatePtr->table = dt + 1; -} - -FORCE_INLINE_TEMPLATE void -ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) -{ - ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; - U32 const nbBits = DInfo.nbBits; - size_t const lowBits = BIT_readBits(bitD, nbBits); - DStatePtr->state = DInfo.nextState + lowBits; -} - -/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum - * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) - * bits before reloading. This value is the maximum number of bytes we read - * after reloading when we are decoding long offets. - */ -#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ - (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ - ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ - : 0) - -typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; - -FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) -{ - seq_t seq; - U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; - U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; - U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; - U32 const totalBits = llBits+mlBits+ofBits; - U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; - U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; - U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; - - /* sequence */ - { size_t offset; - if (!ofBits) - offset = 0; - else { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - } - - if (ofBits <= 1) { - offset += (llBase==0); - if (offset) { - size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { /* offset == 0 */ - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = mlBase - + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - seq.litLength = llBase - + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits/*>0*/) : 0); /* <= 16 bits */ - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); - - DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", - (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); - - /* ANS state update */ - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - return seq; -} - -FORCE_INLINE_TEMPLATE size_t -ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + maxDstSize; - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; - const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); - const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); - const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); - - /* Regen sequences */ - if (nbSeq) { - seqState_t seqState; - dctx->fseEntropy = 1; - { U32 i; for (i=0; ientropy.rep[i]; } - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); - ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { - nbSeq--; - { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); - size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); - DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); - if (ZSTD_isError(oneSeqSize)) return oneSeqSize; - op += oneSeqSize; - } } - - /* check if reached exact end */ - DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); - if (nbSeq) return ERROR(corruption_detected); - /* save reps for next block */ - { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } - } - - /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op-ostart; -} - -static size_t -ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - - - -FORCE_INLINE_TEMPLATE seq_t -ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) -{ - seq_t seq; - U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; - U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; - U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; - U32 const totalBits = llBits+mlBits+ofBits; - U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; - U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; - U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; - - /* sequence */ - { size_t offset; - if (!ofBits) - offset = 0; - else { - ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); - ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); - if (MEM_32bits() && longOffsets) { - U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1); - offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); - if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - } else { - offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); - } - } - - if (ofBits <= 1) { - offset += (llBase==0); - if (offset) { - size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; - temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ - if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset = temp; - } else { - offset = seqState->prevOffset[0]; - } - } else { - seqState->prevOffset[2] = seqState->prevOffset[1]; - seqState->prevOffset[1] = seqState->prevOffset[0]; - seqState->prevOffset[0] = offset; - } - seq.offset = offset; - } - - seq.matchLength = mlBase + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ - if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) - BIT_reloadDStream(&seqState->DStream); - if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) - BIT_reloadDStream(&seqState->DStream); - /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ - ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - - seq.litLength = llBase + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ - if (MEM_32bits()) - BIT_reloadDStream(&seqState->DStream); - - { size_t const pos = seqState->pos + seq.litLength; - const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; - seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. - * No consequence though : no memory access will occur, overly large offset will be detected in ZSTD_execSequenceLong() */ - seqState->pos = pos + seq.matchLength; - } - - /* ANS state update */ - ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ - ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ - if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ - ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ - - return seq; -} - -FORCE_INLINE_TEMPLATE size_t -ZSTD_decompressSequencesLong_body( - ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - const BYTE* ip = (const BYTE*)seqStart; - const BYTE* const iend = ip + seqSize; - BYTE* const ostart = (BYTE* const)dst; - BYTE* const oend = ostart + maxDstSize; - BYTE* op = ostart; - const BYTE* litPtr = dctx->litPtr; - const BYTE* const litEnd = litPtr + dctx->litSize; - const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); - const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); - const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); - - /* Regen sequences */ - if (nbSeq) { -#define STORED_SEQS 4 -#define STOSEQ_MASK (STORED_SEQS-1) -#define ADVANCED_SEQS 4 - seq_t sequences[STORED_SEQS]; - int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); - seqState_t seqState; - int seqNb; - dctx->fseEntropy = 1; - { U32 i; for (i=0; ientropy.rep[i]; } - seqState.prefixStart = prefixStart; - seqState.pos = (size_t)(op-prefixStart); - seqState.dictEnd = dictEnd; - CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); - ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); - ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); - ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); - - /* prepare in advance */ - for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNbentropy.rep[i] = (U32)(seqState.prevOffset[i]); } -#undef STORED_SEQS -#undef STOSEQ_MASK -#undef ADVANCED_SEQS - } - - /* last literal segment */ - { size_t const lastLLSize = litEnd - litPtr; - if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); - memcpy(op, litPtr, lastLLSize); - op += lastLLSize; - } - - return op-ostart; -} - -static size_t -ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - - - -#if DYNAMIC_BMI2 - -static TARGET_ATTRIBUTE("bmi2") size_t -ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - -static TARGET_ATTRIBUTE("bmi2") size_t -ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - -#endif - -typedef size_t (*ZSTD_decompressSequences_t)( - ZSTD_DCtx *dctx, void *dst, size_t maxDstSize, - const void *seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset); - -static size_t ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - DEBUGLOG(5, "ZSTD_decompressSequences"); -#if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); - } -#endif - return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - -static size_t ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, - void* dst, size_t maxDstSize, - const void* seqStart, size_t seqSize, int nbSeq, - const ZSTD_longOffset_e isLongOffset) -{ - DEBUGLOG(5, "ZSTD_decompressSequencesLong"); -#if DYNAMIC_BMI2 - if (dctx->bmi2) { - return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); - } -#endif - return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); -} - -/* ZSTD_getLongOffsetsShare() : - * condition : offTable must be valid - * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) total += 1; - } - - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ - - return total; -} - - -static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, const int frame) -{ /* blockType == blockCompressed */ - const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN))); - DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - - if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); - - /* Decode literals section */ - { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); - if (ZSTD_isError(litCSize)) return litCSize; - ip += litCSize; - srcSize -= litCSize; - } - - /* Build Decoding Tables */ - { int nbSeq; - size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); - if (ZSTD_isError(seqHSize)) return seqHSize; - ip += seqHSize; - srcSize -= seqHSize; - - if ( (!frame || dctx->fParams.windowSize > (1<<24)) - && (nbSeq>0) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - if (shareLongOffsets >= minShare) - return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); - } - - return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); - } -} - - -static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) -{ - if (dst != dctx->previousDstEnd) { /* not contiguous */ - dctx->dictEnd = dctx->previousDstEnd; - dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); - dctx->prefixStart = dst; - dctx->previousDstEnd = dst; - } -} - -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) -{ - size_t dSize; - ZSTD_checkContinuity(dctx, dst); - dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); - dctx->previousDstEnd = (char*)dst + dSize; - return dSize; -} - - -/** ZSTD_insertBlock() : - insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ -ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) -{ - ZSTD_checkContinuity(dctx, blockStart); - dctx->previousDstEnd = (const char*)blockStart + blockSize; - return blockSize; -} - - -static size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE value, size_t length) -{ - if (length > dstCapacity) return ERROR(dstSize_tooSmall); - memset(dst, value, length); - return length; -} - /** ZSTD_findFrameCompressedSize() : * compatible with legacy mode * `src` must point to the start of a ZSTD frame, ZSTD legacy frame, or skippable frame @@ -1806,9 +447,9 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) if (ZSTD_isLegacy(src, srcSize)) return ZSTD_findFrameCompressedSizeLegacy(src, srcSize); #endif - if ( (srcSize >= ZSTD_skippableHeaderSize) - && (MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START ) { - return ZSTD_skippableHeaderSize + MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE); + if ( (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START ) { + return readSkippableFrameSize(src, srcSize); } else { const BYTE* ip = (const BYTE*)src; const BYTE* const ipstart = ip; @@ -1848,8 +489,64 @@ size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) } } + + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + + +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + if (dst == NULL) { + if (srcSize == 0) return 0; + return ERROR(dstBuffer_null); + } + if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + if (dst == NULL) { + if (regenSize == 0) return 0; + return ERROR(dstBuffer_null); + } + if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, b, regenSize); + return regenSize; +} + + /*! ZSTD_decompressFrame() : -* @dctx must be properly initialized */ + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void** srcPtr, size_t *srcSizePtr) @@ -1858,31 +555,33 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, BYTE* const ostart = (BYTE* const)dst; BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; - size_t remainingSize = *srcSizePtr; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); /* check */ - if (remainingSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) + if (remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); /* Frame Header */ - { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_frameHeaderSize_prefix); + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(ip, ZSTD_FRAMEHEADERSIZE_PREFIX); if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; - if (remainingSize < frameHeaderSize+ZSTD_blockHeaderSize) + if (remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); CHECK_F( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) ); - ip += frameHeaderSize; remainingSize -= frameHeaderSize; + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; } /* Loop on each block */ while (1) { size_t decodedSize; blockProperties_t blockProperties; - size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); if (ZSTD_isError(cBlockSize)) return cBlockSize; ip += ZSTD_blockHeaderSize; - remainingSize -= ZSTD_blockHeaderSize; - if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + remainingSrcSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSrcSize) return ERROR(srcSize_wrong); switch(blockProperties.blockType) { @@ -1893,7 +592,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, oend-op, *ip, blockProperties.origSize); break; case bt_reserved : default: @@ -1905,7 +604,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, XXH64_update(&dctx->xxhState, op, decodedSize); op += decodedSize; ip += cBlockSize; - remainingSize -= cBlockSize; + remainingSrcSize -= cBlockSize; if (blockProperties.lastBlock) break; } @@ -1916,16 +615,16 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); U32 checkRead; - if (remainingSize<4) return ERROR(checksum_wrong); + if (remainingSrcSize<4) return ERROR(checksum_wrong); checkRead = MEM_readLE32(ip); if (checkRead != checkCalc) return ERROR(checksum_wrong); ip += 4; - remainingSize -= 4; + remainingSrcSize -= 4; } /* Allow caller to get size read */ *srcPtr = ip; - *srcSizePtr = remainingSize; + *srcSizePtr = remainingSrcSize; return op-ostart; } @@ -1942,11 +641,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ if (ddict) { - dict = ZSTD_DDictDictContent(ddict); - dictSize = ZSTD_DDictDictSize(ddict); + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); } - while (srcSize >= ZSTD_frameHeaderSize_prefix) { + while (srcSize >= ZSTD_FRAMEHEADERSIZE_PREFIX) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) if (ZSTD_isLegacy(src, srcSize)) { @@ -1957,7 +656,9 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, if (dctx->staticSize) return ERROR(memory_allocation); decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + assert(decodedSize <=- dstCapacity); dst = (BYTE*)dst + decodedSize; dstCapacity -= decodedSize; @@ -1970,13 +671,11 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, { U32 const magicNumber = MEM_readLE32(src); DEBUGLOG(4, "reading magic number %08X (expecting %08X)", - (U32)magicNumber, (U32)ZSTD_MAGICNUMBER); - if ((magicNumber & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { - size_t skippableSize; - if (srcSize < ZSTD_skippableHeaderSize) - return ERROR(srcSize_wrong); - skippableSize = MEM_readLE32((const BYTE*)src + ZSTD_FRAMEIDSIZE) - + ZSTD_skippableHeaderSize; + (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) + return skippableSize; if (srcSize < skippableSize) return ERROR(srcSize_wrong); src = (const BYTE *)src + skippableSize; @@ -2010,7 +709,7 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, return ERROR(srcSize_wrong); } if (ZSTD_isError(res)) return res; - /* no need to bound check, ZSTD_decompressFrame already has */ + assert(res <= dstCapacity); dst = (BYTE*)dst + res; dstCapacity -= res; } @@ -2090,9 +789,10 @@ static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skip * or an error code, which can be tested using ZSTD_isError() */ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { - DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (U32)srcSize); + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); /* Sanity check */ - if (srcSize != dctx->expected) return ERROR(srcSize_wrong); /* not allowed */ + if (srcSize != dctx->expected) + return ERROR(srcSize_wrong); /* not allowed */ if (dstCapacity) ZSTD_checkContinuity(dctx, dst); switch (dctx->stage) @@ -2101,9 +801,9 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c assert(src != NULL); if (dctx->format == ZSTD_f_zstd1) { /* allows header */ assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ - if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ memcpy(dctx->headerBuffer, src, srcSize); - dctx->expected = ZSTD_skippableHeaderSize - srcSize; /* remaining to load to get full skippable frame header */ + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ dctx->stage = ZSTDds_decodeSkippableHeader; return 0; } } @@ -2163,19 +863,19 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); break; case bt_rle : - rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); break; case bt_reserved : /* should never happen */ default: return ERROR(corruption_detected); } if (ZSTD_isError(rSize)) return rSize; - DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (U32)rSize); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); dctx->decodedSize += rSize; if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ - DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (U32)dctx->decodedSize); + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { if (dctx->decodedSize != dctx->fParams.frameContentSize) { return ERROR(corruption_detected); @@ -2199,7 +899,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c assert(srcSize == 4); /* guaranteed by dctx->expected */ { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); U32 const check32 = MEM_readLE32(src); - DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", h32, check32); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); if (check32 != h32) return ERROR(checksum_wrong); dctx->expected = 0; dctx->stage = ZSTDds_getFrameHeaderSize; @@ -2208,8 +908,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c case ZSTDds_decodeSkippableHeader: assert(src != NULL); - assert(srcSize <= ZSTD_skippableHeaderSize); - memcpy(dctx->headerBuffer + (ZSTD_skippableHeaderSize - srcSize), src, srcSize); /* complete skippable header */ + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ dctx->stage = ZSTDds_skipFrame; return 0; @@ -2220,7 +920,8 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c return 0; default: - return ERROR(GENERIC); /* impossible */ + assert(0); /* impossible */ + return ERROR(GENERIC); /* some compiler require default to do something */ } } @@ -2234,11 +935,12 @@ static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dict return 0; } -/*! ZSTD_loadEntropy() : +/*! ZSTD_loadDEntropy() : * dict : must point at beginning of a valid zstd dictionary. * @return : size of entropy tables read */ -static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, - const void* const dict, size_t const dictSize) +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) { const BYTE* dictPtr = (const BYTE*)dict; const BYTE* const dictEnd = dictPtr + dictSize; @@ -2252,15 +954,22 @@ static size_t ZSTD_loadEntropy(ZSTD_entropyDTables_t* entropy, ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize); +#else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, workspace, workspaceSize); +#endif if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); dictPtr += hSize; } { short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue = MaxOff, offcodeLog; + unsigned offcodeMaxValue = MaxOff, offcodeLog; size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); if (offcodeMaxValue > MaxOff) return ERROR(dictionary_corrupted); @@ -2320,7 +1029,7 @@ static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); /* load entropy tables */ - { size_t const eSize = ZSTD_loadEntropy(&dctx->entropy, dict, dictSize); + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); dict = (const char*)dict + eSize; dictSize -= eSize; @@ -2364,209 +1073,25 @@ size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t /* ====== ZSTD_DDict ====== */ -struct ZSTD_DDict_s { - void* dictBuffer; - const void* dictContent; - size_t dictSize; - ZSTD_entropyDTables_t entropy; - U32 dictID; - U32 entropyPresent; - ZSTD_customMem cMem; -}; /* typedef'd to ZSTD_DDict within "zstd.h" */ - -static const void* ZSTD_DDictDictContent(const ZSTD_DDict* ddict) -{ - assert(ddict != NULL); - return ddict->dictContent; -} - -static size_t ZSTD_DDictDictSize(const ZSTD_DDict* ddict) -{ - assert(ddict != NULL); - return ddict->dictSize; -} - size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); assert(dctx != NULL); if (ddict) { - dctx->ddictIsCold = (dctx->dictEnd != (const char*)ddict->dictContent + ddict->dictSize); + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); DEBUGLOG(4, "DDict is %s", dctx->ddictIsCold ? "~cold~" : "hot!"); } CHECK_F( ZSTD_decompressBegin(dctx) ); if (ddict) { /* NULL ddict is equivalent to no dictionary */ - dctx->dictID = ddict->dictID; - dctx->prefixStart = ddict->dictContent; - dctx->virtualStart = ddict->dictContent; - dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; - dctx->previousDstEnd = dctx->dictEnd; - if (ddict->entropyPresent) { - dctx->litEntropy = 1; - dctx->fseEntropy = 1; - dctx->LLTptr = ddict->entropy.LLTable; - dctx->MLTptr = ddict->entropy.MLTable; - dctx->OFTptr = ddict->entropy.OFTable; - dctx->HUFptr = ddict->entropy.hufTable; - dctx->entropy.rep[0] = ddict->entropy.rep[0]; - dctx->entropy.rep[1] = ddict->entropy.rep[1]; - dctx->entropy.rep[2] = ddict->entropy.rep[2]; - } else { - dctx->litEntropy = 0; - dctx->fseEntropy = 0; - } + ZSTD_copyDDictParameters(dctx, ddict); } return 0; } -static size_t -ZSTD_loadEntropy_inDDict(ZSTD_DDict* ddict, - ZSTD_dictContentType_e dictContentType) -{ - ddict->dictID = 0; - ddict->entropyPresent = 0; - if (dictContentType == ZSTD_dct_rawContent) return 0; - - if (ddict->dictSize < 8) { - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ - return 0; /* pure content mode */ - } - { U32 const magic = MEM_readLE32(ddict->dictContent); - if (magic != ZSTD_MAGIC_DICTIONARY) { - if (dictContentType == ZSTD_dct_fullDict) - return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ - return 0; /* pure content mode */ - } - } - ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); - - /* load entropy tables */ - CHECK_E( ZSTD_loadEntropy(&ddict->entropy, - ddict->dictContent, ddict->dictSize), - dictionary_corrupted ); - ddict->entropyPresent = 1; - return 0; -} - - -static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType) -{ - if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { - ddict->dictBuffer = NULL; - ddict->dictContent = dict; - if (!dict) dictSize = 0; - } else { - void* const internalBuffer = ZSTD_malloc(dictSize, ddict->cMem); - ddict->dictBuffer = internalBuffer; - ddict->dictContent = internalBuffer; - if (!internalBuffer) return ERROR(memory_allocation); - memcpy(internalBuffer, dict, dictSize); - } - ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ - - /* parse dictionary content */ - CHECK_F( ZSTD_loadEntropy_inDDict(ddict, dictContentType) ); - - return 0; -} - -ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType, - ZSTD_customMem customMem) -{ - if (!customMem.customAlloc ^ !customMem.customFree) return NULL; - - { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); - if (ddict == NULL) return NULL; - ddict->cMem = customMem; - { size_t const initResult = ZSTD_initDDict_internal(ddict, - dict, dictSize, - dictLoadMethod, dictContentType); - if (ZSTD_isError(initResult)) { - ZSTD_freeDDict(ddict); - return NULL; - } } - return ddict; - } -} - -/*! ZSTD_createDDict() : -* Create a digested dictionary, to start decompression without startup delay. -* `dict` content is copied inside DDict. -* Consequently, `dict` can be released after `ZSTD_DDict` creation */ -ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) -{ - ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); -} - -/*! ZSTD_createDDict_byReference() : - * Create a digested dictionary, to start decompression without startup delay. - * Dictionary content is simply referenced, it will be accessed during decompression. - * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ -ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) -{ - ZSTD_customMem const allocator = { NULL, NULL, NULL }; - return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); -} - - -const ZSTD_DDict* ZSTD_initStaticDDict( - void* sBuffer, size_t sBufferSize, - const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, - ZSTD_dictContentType_e dictContentType) -{ - size_t const neededSpace = sizeof(ZSTD_DDict) - + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); - ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; - assert(sBuffer != NULL); - assert(dict != NULL); - if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ - if (sBufferSize < neededSpace) return NULL; - if (dictLoadMethod == ZSTD_dlm_byCopy) { - memcpy(ddict+1, dict, dictSize); /* local copy */ - dict = ddict+1; - } - if (ZSTD_isError( ZSTD_initDDict_internal(ddict, - dict, dictSize, - ZSTD_dlm_byRef, dictContentType) )) - return NULL; - return ddict; -} - - -size_t ZSTD_freeDDict(ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; /* support free on NULL */ - { ZSTD_customMem const cMem = ddict->cMem; - ZSTD_free(ddict->dictBuffer, cMem); - ZSTD_free(ddict, cMem); - return 0; - } -} - -/*! ZSTD_estimateDDictSize() : - * Estimate amount of memory that will be needed to create a dictionary for decompression. - * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ -size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) -{ - return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); -} - -size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; /* support sizeof on NULL */ - return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; -} - /*! ZSTD_getDictID_fromDict() : * Provides the dictID stored within dictionary. * if @return == 0, the dictionary is not conformant with Zstandard specification. @@ -2578,16 +1103,6 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); } -/*! ZSTD_getDictID_fromDDict() : - * Provides the dictID of the dictionary loaded into `ddict`. - * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. - * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ -unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) -{ - if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); -} - /*! ZSTD_getDictID_fromFrame() : * Provides the dictID required to decompresse frame stored within `src`. * If @return == 0, the dictID could not be decoded. @@ -2695,7 +1210,7 @@ size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSiz /* ZSTD_initDStream_usingDict() : - * return : expected size, aka ZSTD_frameHeaderSize_prefix. + * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. * this function cannot fail */ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) { @@ -2703,7 +1218,7 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di zds->streamStage = zdss_init; zds->noForwardProgress = 0; CHECK_F( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) ); - return ZSTD_frameHeaderSize_prefix; + return ZSTD_FRAMEHEADERSIZE_PREFIX; } /* note : this variant can't fail */ @@ -2724,7 +1239,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) } /* ZSTD_resetDStream() : - * return : expected size, aka ZSTD_frameHeaderSize_prefix. + * return : expected size, aka ZSTD_FRAMEHEADERSIZE_PREFIX. * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { @@ -2733,23 +1248,9 @@ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) dctx->lhSize = dctx->inPos = dctx->outStart = dctx->outEnd = 0; dctx->legacyVersion = 0; dctx->hostageByte = 0; - return ZSTD_frameHeaderSize_prefix; + return ZSTD_FRAMEHEADERSIZE_PREFIX; } -size_t ZSTD_setDStreamParameter(ZSTD_DStream* dctx, - ZSTD_DStreamParameter_e paramType, unsigned paramValue) -{ - if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); - switch(paramType) - { - default : return ERROR(parameter_unsupported); - case DStream_p_maxWindowSize : - DEBUGLOG(4, "setting maxWindowSize = %u KB", paramValue >> 10); - dctx->maxWindowSize = paramValue ? paramValue : (U32)(-1); - break; - } - return 0; -} size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) { @@ -2758,18 +1259,92 @@ size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) return 0; } +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) { + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); + if (maxWindowSize < min) return ERROR(parameter_outOfBound); + if (maxWindowSize > max) return ERROR(parameter_outOfBound); dctx->maxWindowSize = maxWindowSize; return 0; } size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) { - DEBUGLOG(4, "ZSTD_DCtx_setFormat : %u", (unsigned)format); + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + if (!ZSTD_dParam_withinBounds(p, v)) \ + return ERROR(parameter_outOfBound); \ +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ if (dctx->streamStage != zdss_init) return ERROR(stage_wrong); - dctx->format = format; + switch(dParam) { + case ZSTD_d_windowLogMax: + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + default:; + } + return ERROR(parameter_unsupported); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + (void)ZSTD_initDStream(dctx); + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + if (dctx->streamStage != zdss_init) + return ERROR(stage_wrong); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + } return 0; } @@ -2799,7 +1374,7 @@ size_t ZSTD_estimateDStreamSize(size_t windowSize) size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) { - U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable */ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ ZSTD_frameHeader zfh; size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); if (ZSTD_isError(err)) return err; @@ -2868,8 +1443,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); if (legacyVersion) { - const void* const dict = zds->ddict ? zds->ddict->dictContent : NULL; - size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; + const void* const dict = zds->ddict ? ZSTD_DDict_dictContent(zds->ddict) : NULL; + size_t const dictSize = zds->ddict ? ZSTD_DDict_dictSize(zds->ddict) : 0; DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); /* legacy support is incompatible with static dctx */ if (zds->staticSize) return ERROR(memory_allocation); @@ -2894,7 +1469,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; - return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + return (MAX(ZSTD_FRAMEHEADERSIZE_MIN, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; @@ -2922,7 +1497,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB DEBUGLOG(4, "Consume header"); CHECK_F(ZSTD_decompressBegin_usingDDict(zds, zds->ddict)); - if ((MEM_readLE32(zds->headerBuffer) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + if ((MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); zds->stage = ZSTDds_skipFrame; } else { @@ -3038,7 +1613,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB someMoreWork = 0; break; - default: return ERROR(GENERIC); /* impossible */ + default: + assert(0); /* impossible */ + return ERROR(GENERIC); /* some compiler require default to do something */ } } /* result */ @@ -3080,13 +1657,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } } - -size_t ZSTD_decompress_generic(ZSTD_DCtx* dctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input) -{ - return ZSTD_decompressStream(dctx, output, input); -} - -size_t ZSTD_decompress_generic_simpleArgs ( +size_t ZSTD_decompressStream_simpleArgs ( ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) @@ -3094,15 +1665,8 @@ size_t ZSTD_decompress_generic_simpleArgs ( ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; ZSTD_inBuffer input = { src, srcSize, *srcPos }; /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompress_generic(dctx, &output, &input); + size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); *dstPos = output.pos; *srcPos = input.pos; return cErr; } - -void ZSTD_DCtx_reset(ZSTD_DCtx* dctx) -{ - (void)ZSTD_initDStream(dctx); - dctx->format = ZSTD_f_zstd1; - dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; -} diff --git a/C/zstd/zstd_decompress_block.c b/C/zstd/zstd_decompress_block.c new file mode 100644 index 00000000..32baad9f --- /dev/null +++ b/C/zstd/zstd_decompress_block.c @@ -0,0 +1,1307 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include /* memcpy, memmove, memset */ +#include "compiler.h" /* prefetch */ +#include "cpu.h" /* bmi2 */ +#include "mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); + return cSize; + } +} + + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize); +/*! ZSTD_decodeLiteralsBlock() : + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); + /* fall-through */ + + case set_compressed: + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; + } + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, dctx->bmi2); + } else { + hufSuccess = HUF_decompress4X_usingDTable_bmi2( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, dctx->bmi2); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace)); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), dctx->bmi2); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), dctx->bmi2); + } + } + + if (HUF_isError(hufSuccess)) return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTD_BLOCKSIZE_MAX) return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } + } +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = (BYTE)nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +void +ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U16 symbolNext[MaxSeq+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u max) return ERROR(corruption_detected); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U32 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + if (!flagRepeatTable) return ERROR(corruption_detected); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSE_isError(headerSize)) return ERROR(corruption_detected); + if (tableLog > maxLog) return ERROR(corruption_detected); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog); + *DTablePtr = DTableSpace; + return headerSize; + } + default : /* impossible */ + assert(0); + return ERROR(GENERIC); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + + /* SeqHead */ + nbSeq = *ip++; + if (!nbSeq) { + *nbSeqPtr=0; + if (srcSize != 1) return ERROR(srcSize_wrong); + return 1; + } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + } else { + if (ip >= iend) return ERROR(srcSize_wrong); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + /* FSE table descriptors */ + if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq); + if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq); + if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq); + if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; + const BYTE* match; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; + const BYTE* prefixStart; + const BYTE* dictEnd; + size_t pos; +} seqState_t; + + +/* ZSTD_execSequenceLast7(): + * exceptional case : decompress a match starting within last 7 bytes of output buffer. + * requires more careful checks, to ensure there is no overflow. + * performance does not matter though. + * note : this case is supposed to be never generated "naturally" by reference encoder, + * since in most cases it needs at least 8 bytes to look for a match. + * but it's allowed by the specification. */ +FORCE_NOINLINE +size_t ZSTD_execSequenceLast7(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must fit within dstBuffer */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* try to read beyond literal buffer */ + + /* copy literals */ + while (op < oLitEnd) *op++ = *(*litPtr)++; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } } + while (op < oMatchEnd) *op++ = *match++; + return sequenceLength; +} + + +HINT_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + if (sequence.offset > (size_t)(oLitEnd - virtualStart)) + return ERROR(corruption_detected); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + + +HINT_INLINE +size_t ZSTD_execSequenceLong(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const dictStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = sequence.match; + + /* check */ + if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd > oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, prefixStart, dictStart, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); /* note : op <= oLitEnd <= oend_w == oend - 8 */ + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - dictStart)) return ERROR(corruption_detected); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + assert(op <= oend_w); + assert(sequence.matchLength >= MINMATCH); + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD) +{ + ZSTD_seqSymbol const DInfo = DStatePtr->table[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most (STREAM_ACCUMULATOR_MIN_32 - 1) + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) +{ + seq_t seq; + U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; + U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; + U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; + U32 const totalBits = llBits+mlBits+ofBits; + U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; + U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; + U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; + + /* sequence */ + { size_t offset; + if (!ofBits) + offset = 0; + else { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + } + + if (ofBits <= 1) { + offset += (llBase==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { /* offset == 0 */ + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = mlBase + + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = llBase + + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits/*>0*/) : 0); /* <= 16 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + /* ANS state update */ + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequences_body( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body"); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { + nbSeq--; + { seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_body: after decode loop, remaining nbSeq : %i", nbSeq); + if (nbSeq) return ERROR(corruption_detected); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequenceLong(seqState_t* seqState, ZSTD_longOffset_e const longOffsets) +{ + seq_t seq; + U32 const llBits = seqState->stateLL.table[seqState->stateLL.state].nbAdditionalBits; + U32 const mlBits = seqState->stateML.table[seqState->stateML.state].nbAdditionalBits; + U32 const ofBits = seqState->stateOffb.table[seqState->stateOffb.state].nbAdditionalBits; + U32 const totalBits = llBits+mlBits+ofBits; + U32 const llBase = seqState->stateLL.table[seqState->stateLL.state].baseValue; + U32 const mlBase = seqState->stateML.table[seqState->stateML.state].baseValue; + U32 const ofBase = seqState->stateOffb.table[seqState->stateOffb.state].baseValue; + + /* sequence */ + { size_t offset; + if (!ofBits) + offset = 0; + else { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + assert(ofBits <= MaxOff); + if (MEM_32bits() && longOffsets) { + U32 const extraBits = ofBits - MIN(ofBits, STREAM_ACCUMULATOR_MIN_32-1); + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + if (MEM_32bits() || extraBits) BIT_reloadDStream(&seqState->DStream); + if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + } + + if (ofBits <= 1) { + offset += (llBase==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = mlBase + ((mlBits>0) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && (totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Verify that there is enough bits to read the rest of the data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + seq.litLength = llBase + ((llBits>0) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + { size_t const pos = seqState->pos + seq.litLength; + const BYTE* const matchBase = (seq.offset > pos) ? seqState->dictEnd : seqState->prefixStart; + seq.match = matchBase + pos - seq.offset; /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : no memory access will occur, overly large offset will be detected in ZSTD_execSequenceLong() */ + seqState->pos = pos + seq.matchLength; + } + + /* ANS state update */ + ZSTD_updateFseState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + ZSTD_updateFseState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 4 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS 4 + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + seqState.prefixStart = prefixStart; + seqState.pos = (size_t)(op-prefixStart); + seqState.dictEnd = dictEnd; + assert(iend >= ip); + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && (seqNbentropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static TARGET_ATTRIBUTE("bmi2") size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (dctx->bmi2) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes mearning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (dctx->bmi2) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +/* ZSTD_getLongOffsetsShare() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) total += 1; + } + + assert(tableLog <= OffFSELog); + total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + + return total; +} +#endif + + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. + * We don't expect that to be the case in 64-bit mode. + * In block mode, window size is not known, so we have to be conservative. + * (note: but it could be evaluated from current-lowLimit) + */ + ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); + DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); + + if (srcSize >= ZSTD_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if ( !usePrefetchDecoder + && (!frame || (dctx->fParams.windowSize > (1<<24))) + && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ + U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (shareLongOffsets >= minShare); + } +#endif + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + } +} + + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, /* frame */ 0); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} diff --git a/C/zstd/zstd_decompress_block.h b/C/zstd/zstd_decompress_block.h new file mode 100644 index 00000000..7e929604 --- /dev/null +++ b/C/zstd/zstd_decompress_block.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include /* size_t */ +#include "zstd.h" /* DCtx, and some public functions */ +#include "zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const int frame); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U32* nbAdditionalBits, + unsigned tableLog); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/C/zstd/zstd_decompress_internal.h b/C/zstd/zstd_decompress_internal.h new file mode 100644 index 00000000..abd00305 --- /dev/null +++ b/C/zstd/zstd_decompress_internal.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "mem.h" /* BYTE, U16, U32 */ +#include "zstd_internal.h" /* ZSTD_seqSymbol */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static const U32 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; + int noForwardProgress; + + /* workspace */ + BYTE litBuffer[ZSTD_BLOCKSIZE_MAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/C/zstd/zstd_double_fast.c b/C/zstd/zstd_double_fast.c index 7b9e18e7..47faf6d6 100644 --- a/C/zstd/zstd_double_fast.c +++ b/C/zstd/zstd_double_fast.c @@ -18,7 +18,7 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashLarge = ms->hashTable; U32 const hBitsL = cParams->hashLog; - U32 const mls = cParams->searchLength; + U32 const mls = cParams->minMatch; U32* const hashSmall = ms->chainTable; U32 const hBitsS = cParams->chainLog; const BYTE* const base = ms->window.base; @@ -309,7 +309,7 @@ size_t ZSTD_compressBlock_doubleFast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - const U32 mls = ms->cParams.searchLength; + const U32 mls = ms->cParams.minMatch; switch(mls) { default: /* includes case 3 */ @@ -329,7 +329,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - const U32 mls = ms->cParams.searchLength; + const U32 mls = ms->cParams.minMatch; switch(mls) { default: /* includes case 3 */ @@ -483,7 +483,7 @@ size_t ZSTD_compressBlock_doubleFast_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) { - U32 const mls = ms->cParams.searchLength; + U32 const mls = ms->cParams.minMatch; switch(mls) { default: /* includes case 3 */ diff --git a/C/zstd/zstd_errors.h b/C/zstd/zstd_errors.h index 57533f28..92a34338 100644 --- a/C/zstd/zstd_errors.h +++ b/C/zstd/zstd_errors.h @@ -72,6 +72,7 @@ typedef enum { ZSTD_error_workSpace_tooSmall= 66, ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, diff --git a/C/zstd/zstd_fast.c b/C/zstd/zstd_fast.c index 24774651..40ba0f73 100644 --- a/C/zstd/zstd_fast.c +++ b/C/zstd/zstd_fast.c @@ -18,7 +18,7 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, const ZSTD_compressionParameters* const cParams = &ms->cParams; U32* const hashTable = ms->hashTable; U32 const hBits = cParams->hashLog; - U32 const mls = cParams->searchLength; + U32 const mls = cParams->minMatch; const BYTE* const base = ms->window.base; const BYTE* ip = base + ms->nextToUpdate; const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; @@ -27,18 +27,18 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, /* Always insert every fastHashFillStep position into the hash table. * Insert the other positions if their hash entry is empty. */ - for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { U32 const current = (U32)(ip - base); - U32 i; - for (i = 0; i < fastHashFillStep; ++i) { - size_t const hash = ZSTD_hashPtr(ip + i, hBits, mls); - if (i == 0 || hashTable[hash] == 0) - hashTable[hash] = current + i; - /* Only load extra positions for ZSTD_dtlm_full */ - if (dtlm == ZSTD_dtlm_fast) - break; - } - } + size_t const hash0 = ZSTD_hashPtr(ip, hBits, mls); + hashTable[hash0] = current; + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hash = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hash] == 0) { /* not yet filled */ + hashTable[hash] = current + p; + } } } } } FORCE_INLINE_TEMPLATE @@ -235,7 +235,7 @@ size_t ZSTD_compressBlock_fast( void const* src, size_t srcSize) { ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->searchLength; + U32 const mls = cParams->minMatch; assert(ms->dictMatchState == NULL); switch(mls) { @@ -256,7 +256,7 @@ size_t ZSTD_compressBlock_fast_dictMatchState( void const* src, size_t srcSize) { ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->searchLength; + U32 const mls = cParams->minMatch; assert(ms->dictMatchState != NULL); switch(mls) { @@ -375,7 +375,7 @@ size_t ZSTD_compressBlock_fast_extDict( void const* src, size_t srcSize) { ZSTD_compressionParameters const* cParams = &ms->cParams; - U32 const mls = cParams->searchLength; + U32 const mls = cParams->minMatch; switch(mls) { default: /* includes case 3 */ diff --git a/C/zstd/zstd_internal.h b/C/zstd/zstd_internal.h index e75adfa6..edeb74b9 100644 --- a/C/zstd/zstd_internal.h +++ b/C/zstd/zstd_internal.h @@ -41,6 +41,9 @@ extern "C" { /* ---- static assert (debug) --- */ #define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError /*-************************************* @@ -75,7 +78,6 @@ static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; #define BIT0 1 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 -#define ZSTD_WINDOWLOG_DEFAULTMAX 27 /* Default maximum allowed window log */ static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; @@ -242,7 +244,7 @@ typedef struct { blockType_e blockType; U32 lastBlock; U32 origSize; -} blockProperties_t; +} blockProperties_t; /* declared here for decompress and fullbench */ /*! ZSTD_getcBlockSize() : * Provides the size of compressed block from block header `src` */ @@ -250,6 +252,13 @@ typedef struct { size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr); +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: decompress, fullbench (does not get its definition from here) */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + + #if defined (__cplusplus) } #endif diff --git a/C/zstd/zstd_lazy.c b/C/zstd/zstd_lazy.c index af615e07..53f998a4 100644 --- a/C/zstd/zstd_lazy.c +++ b/C/zstd/zstd_lazy.c @@ -63,12 +63,13 @@ ZSTD_updateDUBT(ZSTD_matchState_t* ms, static void ZSTD_insertDUBT1(ZSTD_matchState_t* ms, U32 current, const BYTE* inputEnd, - U32 nbCompares, U32 btLow, const ZSTD_dictMode_e dictMode) + U32 nbCompares, U32 btLow, + const ZSTD_dictMode_e dictMode) { const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32* const bt = ms->chainTable; - U32 const btLog = cParams->chainLog - 1; - U32 const btMask = (1 << btLog) - 1; + U32* const bt = ms->chainTable; + U32 const btLog = cParams->chainLog - 1; + U32 const btMask = (1 << btLog) - 1; size_t commonLengthSmaller=0, commonLengthLarger=0; const BYTE* const base = ms->window.base; const BYTE* const dictBase = ms->window.dictBase; @@ -80,7 +81,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, const BYTE* match; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = smallerPtr + 1; - U32 matchIndex = *smallerPtr; + U32 matchIndex = *smallerPtr; /* this candidate is unsorted : next sorted candidate is reached through *smallerPtr, while *largerPtr contains previous unsorted candidate (which is already saved and can be overwritten) */ U32 dummy32; /* to be nullified at the end */ U32 const windowLow = ms->window.lowLimit; @@ -93,6 +94,9 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(matchIndex < current); + /* note : all candidates are now supposed sorted, + * but it's still possible to have nextPtr[1] == ZSTD_DUBT_UNSORTED_MARK + * when a real index has the same value as ZSTD_DUBT_UNSORTED_MARK */ if ( (dictMode != ZSTD_extDict) || (matchIndex+matchLength >= dictLimit) /* both in current segment*/ @@ -108,7 +112,7 @@ ZSTD_insertDUBT1(ZSTD_matchState_t* ms, match = dictBase + matchIndex; matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); if (matchIndex+matchLength >= dictLimit) - match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + match = base + matchIndex; /* preparation for next read of match[matchLength] */ } DEBUGLOG(8, "ZSTD_insertDUBT1: comparing %u with %u : found %u common bytes ", @@ -258,7 +262,7 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, && (nbCandidates > 1) ) { DEBUGLOG(8, "ZSTD_DUBT_findBestMatch: candidate %u is unsorted", matchIndex); - *unsortedMark = previousCandidate; + *unsortedMark = previousCandidate; /* the unsortedMark becomes a reversed chain, to move up back to original position */ previousCandidate = matchIndex; matchIndex = *nextCandidate; nextCandidate = bt + 2*(matchIndex&btMask); @@ -266,11 +270,13 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, nbCandidates --; } + /* nullify last candidate if it's still unsorted + * simplification, detrimental to compression ratio, beneficial for speed */ if ( (matchIndex > unsortLimit) && (*unsortedMark==ZSTD_DUBT_UNSORTED_MARK) ) { DEBUGLOG(7, "ZSTD_DUBT_findBestMatch: nullify last unsorted candidate %u", matchIndex); - *nextCandidate = *unsortedMark = 0; /* nullify next candidate if it's still unsorted (note : simplification, detrimental to compression ratio, beneficial for speed) */ + *nextCandidate = *unsortedMark = 0; } /* batch sort stacked candidates */ @@ -285,14 +291,14 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, } /* find longest match */ - { size_t commonLengthSmaller=0, commonLengthLarger=0; + { size_t commonLengthSmaller = 0, commonLengthLarger = 0; const BYTE* const dictBase = ms->window.dictBase; const U32 dictLimit = ms->window.dictLimit; const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const prefixStart = base + dictLimit; U32* smallerPtr = bt + 2*(current&btMask); U32* largerPtr = bt + 2*(current&btMask) + 1; - U32 matchEndIdx = current+8+1; + U32 matchEndIdx = current + 8 + 1; U32 dummy32; /* to be nullified at the end */ size_t bestLength = 0; @@ -386,7 +392,7 @@ ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) { - switch(ms->cParams.searchLength) + switch(ms->cParams.minMatch) { default : /* includes case 3 */ case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); @@ -402,7 +408,7 @@ static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS ( const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) { - switch(ms->cParams.searchLength) + switch(ms->cParams.minMatch) { default : /* includes case 3 */ case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); @@ -418,7 +424,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) { - switch(ms->cParams.searchLength) + switch(ms->cParams.minMatch) { default : /* includes case 3 */ case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); @@ -433,7 +439,7 @@ static size_t ZSTD_BtFindBestMatch_extDict_selectMLS ( /* ********************************* * Hash Chain ***********************************/ -#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & (mask)] /* Update chains up to ip (excluded) Assumption : always within prefix (i.e. not within extDict) */ @@ -463,7 +469,7 @@ static U32 ZSTD_insertAndFindFirstIndex_internal( U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { const ZSTD_compressionParameters* const cParams = &ms->cParams; - return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.searchLength); + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); } @@ -497,6 +503,7 @@ size_t ZSTD_HcFindBestMatch_generic ( size_t currentMl=0; if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; + assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ if (match[ml] == ip[ml]) /* potentially better */ currentMl = ZSTD_count(ip, match, iLimit); } else { @@ -559,7 +566,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS ( const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) { - switch(ms->cParams.searchLength) + switch(ms->cParams.minMatch) { default : /* includes case 3 */ case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict); @@ -575,7 +582,7 @@ static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS ( const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) { - switch(ms->cParams.searchLength) + switch(ms->cParams.minMatch) { default : /* includes case 3 */ case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState); @@ -591,7 +598,7 @@ FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( const BYTE* ip, const BYTE* const iLimit, size_t* offsetPtr) { - switch(ms->cParams.searchLength) + switch(ms->cParams.minMatch) { default : /* includes case 3 */ case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict); diff --git a/C/zstd/zstd_ldm.c b/C/zstd/zstd_ldm.c index 6238ddec..58eb2ffe 100644 --- a/C/zstd/zstd_ldm.c +++ b/C/zstd/zstd_ldm.c @@ -37,8 +37,8 @@ void ZSTD_ldm_adjustParameters(ldmParams_t* params, params->hashLog = MAX(ZSTD_HASHLOG_MIN, params->windowLog - LDM_HASH_RLOG); assert(params->hashLog <= ZSTD_HASHLOG_MAX); } - if (params->hashEveryLog == 0) { - params->hashEveryLog = params->windowLog < params->hashLog + if (params->hashRateLog == 0) { + params->hashRateLog = params->windowLog < params->hashLog ? 0 : params->windowLog - params->hashLog; } @@ -119,20 +119,20 @@ static void ZSTD_ldm_insertEntry(ldmState_t* ldmState, * * Gets the small hash, checksum, and tag from the rollingHash. * - * If the tag matches (1 << ldmParams.hashEveryLog)-1, then + * If the tag matches (1 << ldmParams.hashRateLog)-1, then * creates an ldmEntry from the offset, and inserts it into the hash table. * * hBits is the length of the small hash, which is the most significant hBits * of rollingHash. The checksum is the next 32 most significant bits, followed - * by ldmParams.hashEveryLog bits that make up the tag. */ + * by ldmParams.hashRateLog bits that make up the tag. */ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState, U64 const rollingHash, U32 const hBits, U32 const offset, ldmParams_t const ldmParams) { - U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashEveryLog); - U32 const tagMask = ((U32)1 << ldmParams.hashEveryLog) - 1; + U32 const tag = ZSTD_ldm_getTag(rollingHash, hBits, ldmParams.hashRateLog); + U32 const tagMask = ((U32)1 << ldmParams.hashRateLog) - 1; if (tag == tagMask) { U32 const hash = ZSTD_ldm_getSmallHash(rollingHash, hBits); U32 const checksum = ZSTD_ldm_getChecksum(rollingHash, hBits); @@ -143,56 +143,6 @@ static void ZSTD_ldm_makeEntryAndInsertByTag(ldmState_t* ldmState, } } -/** ZSTD_ldm_getRollingHash() : - * Get a 64-bit hash using the first len bytes from buf. - * - * Giving bytes s = s_1, s_2, ... s_k, the hash is defined to be - * H(s) = s_1*(a^(k-1)) + s_2*(a^(k-2)) + ... + s_k*(a^0) - * - * where the constant a is defined to be prime8bytes. - * - * The implementation adds an offset to each byte, so - * H(s) = (s_1 + HASH_CHAR_OFFSET)*(a^(k-1)) + ... */ -static U64 ZSTD_ldm_getRollingHash(const BYTE* buf, U32 len) -{ - U64 ret = 0; - U32 i; - for (i = 0; i < len; i++) { - ret *= prime8bytes; - ret += buf[i] + LDM_HASH_CHAR_OFFSET; - } - return ret; -} - -/** ZSTD_ldm_ipow() : - * Return base^exp. */ -static U64 ZSTD_ldm_ipow(U64 base, U64 exp) -{ - U64 ret = 1; - while (exp) { - if (exp & 1) { ret *= base; } - exp >>= 1; - base *= base; - } - return ret; -} - -U64 ZSTD_ldm_getHashPower(U32 minMatchLength) { - DEBUGLOG(4, "ZSTD_ldm_getHashPower: mml=%u", minMatchLength); - assert(minMatchLength >= ZSTD_LDM_MINMATCH_MIN); - return ZSTD_ldm_ipow(prime8bytes, minMatchLength - 1); -} - -/** ZSTD_ldm_updateHash() : - * Updates hash by removing toRemove and adding toAdd. */ -static U64 ZSTD_ldm_updateHash(U64 hash, BYTE toRemove, BYTE toAdd, U64 hashPower) -{ - hash -= ((toRemove + LDM_HASH_CHAR_OFFSET) * hashPower); - hash *= prime8bytes; - hash += toAdd + LDM_HASH_CHAR_OFFSET; - return hash; -} - /** ZSTD_ldm_countBackwardsMatch() : * Returns the number of bytes that match backwards before pIn and pMatch. * @@ -238,6 +188,7 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, case ZSTD_btlazy2: case ZSTD_btopt: case ZSTD_btultra: + case ZSTD_btultra2: break; default: assert(0); /* not possible : not a valid strategy id */ @@ -261,9 +212,9 @@ static U64 ZSTD_ldm_fillLdmHashTable(ldmState_t* state, const BYTE* cur = lastHashed + 1; while (cur < iend) { - rollingHash = ZSTD_ldm_updateHash(rollingHash, cur[-1], - cur[ldmParams.minMatchLength-1], - state->hashPower); + rollingHash = ZSTD_rollingHash_rotate(rollingHash, cur[-1], + cur[ldmParams.minMatchLength-1], + state->hashPower); ZSTD_ldm_makeEntryAndInsertByTag(state, rollingHash, hBits, (U32)(cur - base), ldmParams); @@ -297,8 +248,8 @@ static size_t ZSTD_ldm_generateSequences_internal( U64 const hashPower = ldmState->hashPower; U32 const hBits = params->hashLog - params->bucketSizeLog; U32 const ldmBucketSize = 1U << params->bucketSizeLog; - U32 const hashEveryLog = params->hashEveryLog; - U32 const ldmTagMask = (1U << params->hashEveryLog) - 1; + U32 const hashRateLog = params->hashRateLog; + U32 const ldmTagMask = (1U << params->hashRateLog) - 1; /* Prefix and extDict parameters */ U32 const dictLimit = ldmState->window.dictLimit; U32 const lowestIndex = extDict ? ldmState->window.lowLimit : dictLimit; @@ -324,16 +275,16 @@ static size_t ZSTD_ldm_generateSequences_internal( size_t forwardMatchLength = 0, backwardMatchLength = 0; ldmEntry_t* bestEntry = NULL; if (ip != istart) { - rollingHash = ZSTD_ldm_updateHash(rollingHash, lastHashed[0], - lastHashed[minMatchLength], - hashPower); + rollingHash = ZSTD_rollingHash_rotate(rollingHash, lastHashed[0], + lastHashed[minMatchLength], + hashPower); } else { - rollingHash = ZSTD_ldm_getRollingHash(ip, minMatchLength); + rollingHash = ZSTD_rollingHash_compute(ip, minMatchLength); } lastHashed = ip; /* Do not insert and do not look for a match */ - if (ZSTD_ldm_getTag(rollingHash, hBits, hashEveryLog) != ldmTagMask) { + if (ZSTD_ldm_getTag(rollingHash, hBits, hashRateLog) != ldmTagMask) { ip++; continue; } @@ -593,7 +544,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, void const* src, size_t srcSize) { const ZSTD_compressionParameters* const cParams = &ms->cParams; - unsigned const minMatch = cParams->searchLength; + unsigned const minMatch = cParams->minMatch; ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(cParams->strategy, ZSTD_matchState_dictMode(ms)); /* Input bounds */ diff --git a/C/zstd/zstd_ldm.h b/C/zstd/zstd_ldm.h index 21fba4d5..a4784612 100644 --- a/C/zstd/zstd_ldm.h +++ b/C/zstd/zstd_ldm.h @@ -21,7 +21,7 @@ extern "C" { * Long distance matching ***************************************/ -#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_DEFAULTMAX +#define ZSTD_LDM_DEFAULT_WINDOW_LOG ZSTD_WINDOWLOG_LIMIT_DEFAULT /** * ZSTD_ldm_generateSequences(): @@ -86,12 +86,8 @@ size_t ZSTD_ldm_getTableSize(ldmParams_t params); */ size_t ZSTD_ldm_getMaxNbSeq(ldmParams_t params, size_t maxChunkSize); -/** ZSTD_ldm_getTableSize() : - * Return prime8bytes^(minMatchLength-1) */ -U64 ZSTD_ldm_getHashPower(U32 minMatchLength); - /** ZSTD_ldm_adjustParameters() : - * If the params->hashEveryLog is not set, set it to its default value based on + * If the params->hashRateLog is not set, set it to its default value based on * windowLog and params->hashLog. * * Ensures that params->bucketSizeLog is <= params->hashLog (setting it to diff --git a/C/zstd/zstd_opt.c b/C/zstd/zstd_opt.c index 8af69a91..44de6e97 100644 --- a/C/zstd/zstd_opt.c +++ b/C/zstd/zstd_opt.c @@ -17,6 +17,8 @@ #define ZSTD_FREQ_DIV 4 /* log factor when using previous stats to init next stats */ #define ZSTD_MAX_PRICE (1<<30) +#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ + /*-************************************* * Price functions for optimal parser @@ -52,11 +54,15 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) return weight; } -/* debugging function, @return price in bytes */ +#if (DEBUGLEVEL>=2) +/* debugging function, + * @return price in bytes as fractional value + * for debug messages only */ MEM_STATIC double ZSTD_fCost(U32 price) { return (double)price / (BITCOST_MULTIPLIER*8); } +#endif static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) { @@ -67,29 +73,44 @@ static void ZSTD_setBasePrices(optState_t* optPtr, int optLevel) } -static U32 ZSTD_downscaleStat(U32* table, U32 lastEltIndex, int malus) +/* ZSTD_downscaleStat() : + * reduce all elements in table by a factor 2^(ZSTD_FREQ_DIV+malus) + * return the resulting sum of elements */ +static U32 ZSTD_downscaleStat(unsigned* table, U32 lastEltIndex, int malus) { U32 s, sum=0; + DEBUGLOG(5, "ZSTD_downscaleStat (nbElts=%u)", (unsigned)lastEltIndex+1); assert(ZSTD_FREQ_DIV+malus > 0 && ZSTD_FREQ_DIV+malus < 31); - for (s=0; s<=lastEltIndex; s++) { + for (s=0; s> (ZSTD_FREQ_DIV+malus)); sum += table[s]; } return sum; } -static void ZSTD_rescaleFreqs(optState_t* const optPtr, - const BYTE* const src, size_t const srcSize, - int optLevel) +/* ZSTD_rescaleFreqs() : + * if first block (detected by optPtr->litLengthSum == 0) : init statistics + * take hints from dictionary if there is one + * or init from zero, using src for literals stats, or flat 1 for match symbols + * otherwise downscale existing stats, to be used as seed for next block. + */ +static void +ZSTD_rescaleFreqs(optState_t* const optPtr, + const BYTE* const src, size_t const srcSize, + int const optLevel) { + DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); optPtr->priceType = zop_dynamic; if (optPtr->litLengthSum == 0) { /* first block : init */ - if (srcSize <= 1024) /* heuristic */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ + DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); optPtr->priceType = zop_predef; + } assert(optPtr->symbolCosts != NULL); - if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { /* huffman table presumed generated by dictionary */ + if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { + /* huffman table presumed generated by dictionary */ optPtr->priceType = zop_dynamic; assert(optPtr->litFreq != NULL); @@ -208,7 +229,9 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP /* dynamic statistics */ { U32 const llCode = ZSTD_LLcode(litLength); - return (LL_bits[llCode] * BITCOST_MULTIPLIER) + (optPtr->litLengthSumBasePrice - WEIGHT(optPtr->litLengthFreq[llCode], optLevel)); + return (LL_bits[llCode] * BITCOST_MULTIPLIER) + + optPtr->litLengthSumBasePrice + - WEIGHT(optPtr->litLengthFreq[llCode], optLevel); } } @@ -253,7 +276,7 @@ static int ZSTD_literalsContribution(const BYTE* const literals, U32 const litLe FORCE_INLINE_TEMPLATE U32 ZSTD_getMatchPrice(U32 const offset, U32 const matchLength, - const optState_t* const optPtr, + const optState_t* const optPtr, int const optLevel) { U32 price; @@ -385,7 +408,6 @@ static U32 ZSTD_insertBt1( U32* largerPtr = smallerPtr + 1; U32 dummy32; /* to be nullified at the end */ U32 const windowLow = ms->window.lowLimit; - U32 const matchLow = windowLow ? windowLow : 1; U32 matchEndIdx = current+8+1; size_t bestLength = 8; U32 nbCompares = 1U << cParams->searchLog; @@ -401,7 +423,8 @@ static U32 ZSTD_insertBt1( assert(ip <= iend-8); /* required for h calculation */ hashTable[h] = current; /* Update Hash Table */ - while (nbCompares-- && (matchIndex >= matchLow)) { + assert(windowLow > 0); + while (nbCompares-- && (matchIndex >= windowLow)) { U32* const nextPtr = bt + 2*(matchIndex & btMask); size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ assert(matchIndex < current); @@ -479,7 +502,7 @@ void ZSTD_updateTree_internal( const BYTE* const base = ms->window.base; U32 const target = (U32)(ip - base); U32 idx = ms->nextToUpdate; - DEBUGLOG(5, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", + DEBUGLOG(6, "ZSTD_updateTree_internal, from %u to %u (dictMode:%u)", idx, target, dictMode); while(idx < target) @@ -488,15 +511,18 @@ void ZSTD_updateTree_internal( } void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { - ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.searchLength, ZSTD_noDict); + ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); } FORCE_INLINE_TEMPLATE U32 ZSTD_insertBtAndGetAllMatches ( ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - U32 rep[ZSTD_REP_NUM], U32 const ll0, - ZSTD_match_t* matches, const U32 lengthToBeat, U32 const mls /* template */) + U32 rep[ZSTD_REP_NUM], + U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + ZSTD_match_t* matches, + const U32 lengthToBeat, + U32 const mls /* template */) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); @@ -542,6 +568,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( DEBUGLOG(8, "ZSTD_insertBtAndGetAllMatches: current=%u", current); /* check repCode */ + assert(ll0 <= 1); /* necessarily 1 or 0 */ { U32 const lastR = ZSTD_REP_NUM + ll0; U32 repCode; for (repCode = ll0; repCode < lastR; repCode++) { @@ -724,7 +751,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches ( ZSTD_match_t* matches, U32 const lengthToBeat) { const ZSTD_compressionParameters* const cParams = &ms->cParams; - U32 const matchLengthSearch = cParams->searchLength; + U32 const matchLengthSearch = cParams->minMatch; DEBUGLOG(8, "ZSTD_BtGetAllMatches"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateTree_internal(ms, ip, iHighLimit, matchLengthSearch, dictMode); @@ -774,12 +801,30 @@ static U32 ZSTD_totalLen(ZSTD_optimal_t sol) return sol.litlen + sol.mlen; } +#if 0 /* debug */ + +static void +listStats(const U32* table, int lastEltID) +{ + int const nbElts = lastEltID + 1; + int enb; + for (enb=0; enb < nbElts; enb++) { + (void)table; + //RAWLOG(2, "%3i:%3i, ", enb, table[enb]); + RAWLOG(2, "%4i,", table[enb]); + } + RAWLOG(2, " \n"); +} + +#endif + FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], - const void* src, size_t srcSize, - const int optLevel, const ZSTD_dictMode_e dictMode) + const void* src, size_t srcSize, + const int optLevel, + const ZSTD_dictMode_e dictMode) { optState_t* const optStatePtr = &ms->opt; const BYTE* const istart = (const BYTE*)src; @@ -792,14 +837,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); - U32 const minMatch = (cParams->searchLength == 3) ? 3 : 4; + U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4; ZSTD_optimal_t* const opt = optStatePtr->priceTable; ZSTD_match_t* const matches = optStatePtr->matchTable; ZSTD_optimal_t lastSequence; /* init */ - DEBUGLOG(5, "ZSTD_compressBlock_opt_generic"); + DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u", + (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate); assert(optLevel <= 2); ms->nextToUpdate3 = ms->nextToUpdate; ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel); @@ -999,7 +1045,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ U32 const offCode = opt[storePos].off; U32 const advance = llen + mlen; DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", - anchor - istart, llen, mlen); + anchor - istart, (unsigned)llen, (unsigned)mlen); if (mlen==0) { /* only literals => must be last "sequence", actually starting a new stream of sequences */ assert(storePos == storeEnd); /* must be last sequence */ @@ -1047,11 +1093,11 @@ size_t ZSTD_compressBlock_btopt( /* used in 2-pass strategy */ -static U32 ZSTD_upscaleStat(U32* table, U32 lastEltIndex, int bonus) +static U32 ZSTD_upscaleStat(unsigned* table, U32 lastEltIndex, int bonus) { U32 s, sum=0; - assert(ZSTD_FREQ_DIV+bonus > 0); - for (s=0; s<=lastEltIndex; s++) { + assert(ZSTD_FREQ_DIV+bonus >= 0); + for (s=0; slitSum = ZSTD_upscaleStat(optPtr->litFreq, MaxLit, 0); - optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 1); - optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 1); - optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 1); + optPtr->litLengthSum = ZSTD_upscaleStat(optPtr->litLengthFreq, MaxLL, 0); + optPtr->matchLengthSum = ZSTD_upscaleStat(optPtr->matchLengthFreq, MaxML, 0); + optPtr->offCodeSum = ZSTD_upscaleStat(optPtr->offCodeFreq, MaxOff, 0); +} + +/* ZSTD_initStats_ultra(): + * make a first compression pass, just to seed stats with more accurate starting values. + * only works on first block, with no dictionary and no ldm. + * this function cannot error, hence its constract must be respected. + */ +static void +ZSTD_initStats_ultra(ZSTD_matchState_t* ms, + seqStore_t* seqStore, + U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 tmpRep[ZSTD_REP_NUM]; /* updated rep codes will sink here */ + memcpy(tmpRep, rep, sizeof(tmpRep)); + + DEBUGLOG(4, "ZSTD_initStats_ultra (srcSize=%zu)", srcSize); + assert(ms->opt.litLengthSum == 0); /* first block */ + assert(seqStore->sequences == seqStore->sequencesStart); /* no ldm */ + assert(ms->window.dictLimit == ms->window.lowLimit); /* no dictionary */ + assert(ms->window.dictLimit - ms->nextToUpdate <= 1); /* no prefix (note: intentional overflow, defined as 2-complement) */ + + ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ + + /* invalidate first scan from history */ + ZSTD_resetSeqStore(seqStore); + ms->window.base -= srcSize; + ms->window.dictLimit += (U32)srcSize; + ms->window.lowLimit = ms->window.dictLimit; + ms->nextToUpdate = ms->window.dictLimit; + ms->nextToUpdate3 = ms->window.dictLimit; + + /* re-inforce weight of collected statistics */ + ZSTD_upscaleStats(&ms->opt); } size_t ZSTD_compressBlock_btultra( @@ -1073,33 +1153,34 @@ size_t ZSTD_compressBlock_btultra( const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock_btultra (srcSize=%zu)", srcSize); -#if 0 - /* 2-pass strategy (disabled) + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); +} + +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + const void* src, size_t srcSize) +{ + U32 const current = (U32)((const BYTE*)src - ms->window.base); + DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); + + /* 2-pass strategy: * this strategy makes a first pass over first block to collect statistics * and seed next round's statistics with it. + * After 1st pass, function forgets everything, and starts a new block. + * Consequently, this can only work if no data has been previously loaded in tables, + * aka, no dictionary, no prefix, no ldm preprocessing. * The compression ratio gain is generally small (~0.5% on first block), * the cost is 2x cpu time on first block. */ assert(srcSize <= ZSTD_BLOCKSIZE_MAX); if ( (ms->opt.litLengthSum==0) /* first block */ - && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ - && (ms->window.dictLimit == ms->window.lowLimit) ) { /* no dictionary */ - U32 tmpRep[ZSTD_REP_NUM]; - DEBUGLOG(5, "ZSTD_compressBlock_btultra: first block: collecting statistics"); - assert(ms->nextToUpdate >= ms->window.dictLimit - && ms->nextToUpdate <= ms->window.dictLimit + 1); - memcpy(tmpRep, rep, sizeof(tmpRep)); - ZSTD_compressBlock_opt_generic(ms, seqStore, tmpRep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); /* generate stats into ms->opt*/ - ZSTD_resetSeqStore(seqStore); - /* invalidate first scan from history */ - ms->window.base -= srcSize; - ms->window.dictLimit += (U32)srcSize; - ms->window.lowLimit = ms->window.dictLimit; - ms->nextToUpdate = ms->window.dictLimit; - ms->nextToUpdate3 = ms->window.dictLimit; - /* re-inforce weight of collected statistics */ - ZSTD_upscaleStats(&ms->opt); + && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ + && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ + && (current == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) + ) { + ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); } -#endif + return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_noDict); } @@ -1130,3 +1211,7 @@ size_t ZSTD_compressBlock_btultra_extDict( { return ZSTD_compressBlock_opt_generic(ms, seqStore, rep, src, srcSize, 2 /*optLevel*/, ZSTD_extDict); } + +/* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ diff --git a/C/zstd/zstd_opt.h b/C/zstd/zstd_opt.h index eeadb604..094f7476 100644 --- a/C/zstd/zstd_opt.h +++ b/C/zstd/zstd_opt.h @@ -26,6 +26,10 @@ size_t ZSTD_compressBlock_btopt( size_t ZSTD_compressBlock_btultra( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); +size_t ZSTD_compressBlock_btultra2( + ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], + void const* src, size_t srcSize); + size_t ZSTD_compressBlock_btopt_dictMatchState( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], @@ -41,6 +45,10 @@ size_t ZSTD_compressBlock_btultra_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); + /* note : no btultra2 variant for extDict nor dictMatchState, + * because btultra2 is not meant to work with dictionaries + * and is only specific for the first block (no prefix) */ + #if defined (__cplusplus) } #endif diff --git a/C/zstd/zstd_v04.c b/C/zstd/zstd_v04.c index e852bb91..d7522025 100644 --- a/C/zstd/zstd_v04.c +++ b/C/zstd/zstd_v04.c @@ -240,17 +240,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) /* ************************************* * Types ***************************************/ -#define ZSTD_WINDOWLOG_MAX 26 -#define ZSTD_WINDOWLOG_MIN 18 #define ZSTD_WINDOWLOG_ABSOLUTEMIN 11 -#define ZSTD_CONTENTLOG_MAX (ZSTD_WINDOWLOG_MAX+1) -#define ZSTD_CONTENTLOG_MIN 4 -#define ZSTD_HASHLOG_MAX 28 -#define ZSTD_HASHLOG_MIN 4 -#define ZSTD_SEARCHLOG_MAX (ZSTD_CONTENTLOG_MAX-1) -#define ZSTD_SEARCHLOG_MIN 1 -#define ZSTD_SEARCHLENGTH_MAX 7 -#define ZSTD_SEARCHLENGTH_MIN 4 /** from faster to stronger */ typedef enum { ZSTD_fast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2 } ZSTD_strategy; diff --git a/C/zstd/zstd_v05.c b/C/zstd/zstd_v05.c index a1580a27..96bffc44 100644 --- a/C/zstd/zstd_v05.c +++ b/C/zstd/zstd_v05.c @@ -836,7 +836,7 @@ MEM_STATIC void BITv05_skipBits(BITv05_DStream_t* bitD, U32 nbBits) bitD->bitsConsumed += nbBits; } -MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits) +MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits) { size_t value = BITv05_lookBits(bitD, nbBits); BITv05_skipBits(bitD, nbBits); @@ -845,7 +845,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, U32 nbBits) /*!BITv05_readBitsFast : * unsafe version; only works only if nbBits >= 1 */ -MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, U32 nbBits) +MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits) { size_t value = BITv05_lookBitsFast(bitD, nbBits); BITv05_skipBits(bitD, nbBits); @@ -1162,7 +1162,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr) /* ************************************************************** * Complex types ****************************************************************/ -typedef U32 DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)]; +typedef unsigned DTable_max_t[FSEv05_DTABLE_SIZE_U32(FSEv05_MAX_TABLELOG)]; /* ************************************************************** @@ -2191,7 +2191,7 @@ static void HUFv05_fillDTableX4(HUFv05_DEltX4* DTable, const U32 targetLog, } } -size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize) +size_t HUFv05_readDTableX4 (unsigned* DTable, const void* src, size_t srcSize) { BYTE weightList[HUFv05_MAX_SYMBOL_VALUE + 1]; sortedSymbol_t sortedSymbol[HUFv05_MAX_SYMBOL_VALUE + 1]; @@ -2205,7 +2205,7 @@ size_t HUFv05_readDTableX4 (U32* DTable, const void* src, size_t srcSize) void* dtPtr = DTable; HUFv05_DEltX4* const dt = ((HUFv05_DEltX4*)dtPtr) + 1; - HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(U32)); /* if compilation fails here, assertion is false */ + HUFv05_STATIC_ASSERT(sizeof(HUFv05_DEltX4) == sizeof(unsigned)); /* if compilation fails here, assertion is false */ if (memLog > HUFv05_ABSOLUTEMAX_TABLELOG) return ERROR(tableLog_tooLarge); //memset(weightList, 0, sizeof(weightList)); /* is not necessary, even though some analyzer complain ... */ @@ -2332,7 +2332,7 @@ static inline size_t HUFv05_decodeStreamX4(BYTE* p, BITv05_DStream_t* bitDPtr, B size_t HUFv05_decompress1X4_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U32* DTable) + const unsigned* DTable) { const BYTE* const istart = (const BYTE*) cSrc; BYTE* const ostart = (BYTE*) dst; @@ -2375,7 +2375,7 @@ size_t HUFv05_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t size_t HUFv05_decompress4X4_usingDTable( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const U32* DTable) + const unsigned* DTable) { if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ @@ -2999,7 +2999,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t const BYTE* ip = istart; const BYTE* const iend = istart + srcSize; U32 LLtype, Offtype, MLtype; - U32 LLlog, Offlog, MLlog; + unsigned LLlog, Offlog, MLlog; size_t dumpsLength; /* check */ @@ -3057,7 +3057,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ - { U32 max = MaxLL; + { unsigned max = MaxLL; headerSize = FSEv05_readNCount(norm, &max, &LLlog, ip, iend-ip); if (FSEv05_isError(headerSize)) return ERROR(GENERIC); if (LLlog > LLFSEv05Log) return ERROR(corruption_detected); @@ -3081,7 +3081,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ - { U32 max = MaxOff; + { unsigned max = MaxOff; headerSize = FSEv05_readNCount(norm, &max, &Offlog, ip, iend-ip); if (FSEv05_isError(headerSize)) return ERROR(GENERIC); if (Offlog > OffFSEv05Log) return ERROR(corruption_detected); @@ -3105,7 +3105,7 @@ static size_t ZSTDv05_decodeSeqHeaders(int* nbSeq, const BYTE** dumpsPtr, size_t break; case FSEv05_ENCODING_DYNAMIC : default : /* impossible */ - { U32 max = MaxML; + { unsigned max = MaxML; headerSize = FSEv05_readNCount(norm, &max, &MLlog, ip, iend-ip); if (FSEv05_isError(headerSize)) return ERROR(GENERIC); if (MLlog > MLFSEv05Log) return ERROR(corruption_detected); @@ -3305,9 +3305,9 @@ static size_t ZSTDv05_decompressSequences( const BYTE* const litEnd = litPtr + dctx->litSize; int nbSeq=0; const BYTE* dumps = NULL; - U32* DTableLL = dctx->LLTable; - U32* DTableML = dctx->MLTable; - U32* DTableOffb = dctx->OffTable; + unsigned* DTableLL = dctx->LLTable; + unsigned* DTableML = dctx->MLTable; + unsigned* DTableOffb = dctx->OffTable; const BYTE* const base = (const BYTE*) (dctx->base); const BYTE* const vBase = (const BYTE*) (dctx->vBase); const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); @@ -3633,7 +3633,7 @@ static size_t ZSTDv05_loadEntropy(ZSTDv05_DCtx* dctx, const void* dict, size_t d { size_t hSize, offcodeHeaderSize, matchlengthHeaderSize, errorCode, litlengthHeaderSize; short offcodeNCount[MaxOff+1]; - U32 offcodeMaxValue=MaxOff, offcodeLog; + unsigned offcodeMaxValue=MaxOff, offcodeLog; short matchlengthNCount[MaxML+1]; unsigned matchlengthMaxValue = MaxML, matchlengthLog; short litlengthNCount[MaxLL+1]; diff --git a/C/zstd/zstdmt_compress.c b/C/zstd/zstdmt_compress.c index f4aba1d2..2cbd6ffa 100644 --- a/C/zstd/zstdmt_compress.c +++ b/C/zstd/zstdmt_compress.c @@ -9,21 +9,19 @@ */ -/* ====== Tuning parameters ====== */ -#define ZSTDMT_NBWORKERS_MAX 200 -#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (2 GB)) /* note : limited by `jobSize` type, which is `unsigned` */ -#define ZSTDMT_OVERLAPLOG_DEFAULT 6 - - /* ====== Compiler specifics ====== */ #if defined(_MSC_VER) # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ #endif +/* ====== Constants ====== */ +#define ZSTDMT_OVERLAPLOG_DEFAULT 0 + + /* ====== Dependencies ====== */ #include /* memcpy, memset */ -#include /* INT_MAX */ +#include /* INT_MAX, UINT_MAX */ #include "pool.h" /* threadpool */ #include "threading.h" /* mutex */ #include "zstd_compress_internal.h" /* MIN, ERROR, ZSTD_*, ZSTD_highbit32 */ @@ -57,9 +55,9 @@ static unsigned long long GetCurrentClockTimeMicroseconds(void) static clock_t _ticksPerSecond = 0; if (_ticksPerSecond <= 0) _ticksPerSecond = sysconf(_SC_CLK_TCK); - { struct tms junk; clock_t newTicks = (clock_t) times(&junk); - return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); } -} + { struct tms junk; clock_t newTicks = (clock_t) times(&junk); + return ((((unsigned long long)newTicks)*(1000000))/_ticksPerSecond); +} } #define MUTEX_WAIT_TIME_DLEVEL 6 #define ZSTD_PTHREAD_MUTEX_LOCK(mutex) { \ @@ -342,8 +340,8 @@ static ZSTDMT_seqPool* ZSTDMT_expandSeqPool(ZSTDMT_seqPool* pool, U32 nbWorkers) typedef struct { ZSTD_pthread_mutex_t poolMutex; - unsigned totalCCtx; - unsigned availCCtx; + int totalCCtx; + int availCCtx; ZSTD_customMem cMem; ZSTD_CCtx* cctx[1]; /* variable size */ } ZSTDMT_CCtxPool; @@ -351,16 +349,16 @@ typedef struct { /* note : all CCtx borrowed from the pool should be released back to the pool _before_ freeing the pool */ static void ZSTDMT_freeCCtxPool(ZSTDMT_CCtxPool* pool) { - unsigned u; - for (u=0; utotalCCtx; u++) - ZSTD_freeCCtx(pool->cctx[u]); /* note : compatible with free on NULL */ + int cid; + for (cid=0; cidtotalCCtx; cid++) + ZSTD_freeCCtx(pool->cctx[cid]); /* note : compatible with free on NULL */ ZSTD_pthread_mutex_destroy(&pool->poolMutex); ZSTD_free(pool, pool->cMem); } /* ZSTDMT_createCCtxPool() : * implies nbWorkers >= 1 , checked by caller ZSTDMT_createCCtx() */ -static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers, +static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(int nbWorkers, ZSTD_customMem cMem) { ZSTDMT_CCtxPool* const cctxPool = (ZSTDMT_CCtxPool*) ZSTD_calloc( @@ -381,7 +379,7 @@ static ZSTDMT_CCtxPool* ZSTDMT_createCCtxPool(unsigned nbWorkers, } static ZSTDMT_CCtxPool* ZSTDMT_expandCCtxPool(ZSTDMT_CCtxPool* srcPool, - unsigned nbWorkers) + int nbWorkers) { if (srcPool==NULL) return NULL; if (nbWorkers <= srcPool->totalCCtx) return srcPool; /* good enough */ @@ -469,9 +467,9 @@ static int ZSTDMT_serialState_reset(serialState_t* serialState, ZSTDMT_seqPool* DEBUGLOG(4, "LDM window size = %u KB", (1U << params.cParams.windowLog) >> 10); ZSTD_ldm_adjustParameters(¶ms.ldmParams, ¶ms.cParams); assert(params.ldmParams.hashLog >= params.ldmParams.bucketSizeLog); - assert(params.ldmParams.hashEveryLog < 32); + assert(params.ldmParams.hashRateLog < 32); serialState->ldmState.hashPower = - ZSTD_ldm_getHashPower(params.ldmParams.minMatchLength); + ZSTD_rollingHash_primePower(params.ldmParams.minMatchLength); } else { memset(¶ms.ldmParams, 0, sizeof(params.ldmParams)); } @@ -674,7 +672,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) if (ZSTD_isError(initError)) JOB_ERROR(initError); } else { /* srcStart points at reloaded section */ U64 const pledgedSrcSize = job->firstJob ? job->fullFrameSize : job->src.size; - { size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_p_forceMaxWindow, !job->firstJob); + { size_t const forceWindowError = ZSTD_CCtxParam_setParameter(&jobParams, ZSTD_c_forceMaxWindow, !job->firstJob); if (ZSTD_isError(forceWindowError)) JOB_ERROR(forceWindowError); } { size_t const initError = ZSTD_compressBegin_advanced_internal(cctx, @@ -777,6 +775,14 @@ typedef struct { static const roundBuff_t kNullRoundBuff = {NULL, 0, 0}; +#define RSYNC_LENGTH 32 + +typedef struct { + U64 hash; + U64 hitMask; + U64 primePower; +} rsyncState_t; + struct ZSTDMT_CCtx_s { POOL_ctx* factory; ZSTDMT_jobDescription* jobs; @@ -790,6 +796,7 @@ struct ZSTDMT_CCtx_s { inBuff_t inBuff; roundBuff_t roundBuff; serialState_t serial; + rsyncState_t rsync; unsigned singleBlockingThread; unsigned jobIDMask; unsigned doneJobID; @@ -859,7 +866,7 @@ size_t ZSTDMT_CCtxParam_setNbWorkers(ZSTD_CCtx_params* params, unsigned nbWorker { if (nbWorkers > ZSTDMT_NBWORKERS_MAX) nbWorkers = ZSTDMT_NBWORKERS_MAX; params->nbWorkers = nbWorkers; - params->overlapSizeLog = ZSTDMT_OVERLAPLOG_DEFAULT; + params->overlapLog = ZSTDMT_OVERLAPLOG_DEFAULT; params->jobSize = 0; return nbWorkers; } @@ -969,52 +976,59 @@ size_t ZSTDMT_sizeof_CCtx(ZSTDMT_CCtx* mtctx) } /* Internal only */ -size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, - ZSTDMT_parameter parameter, unsigned value) { +size_t +ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, + ZSTDMT_parameter parameter, + int value) +{ DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter"); switch(parameter) { case ZSTDMT_p_jobSize : - DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %u", value); - if ( (value > 0) /* value==0 => automatic job size */ - & (value < ZSTDMT_JOBSIZE_MIN) ) + DEBUGLOG(4, "ZSTDMT_CCtxParam_setMTCtxParameter : set jobSize to %i", value); + if ( value != 0 /* default */ + && value < ZSTDMT_JOBSIZE_MIN) value = ZSTDMT_JOBSIZE_MIN; - if (value > ZSTDMT_JOBSIZE_MAX) - value = ZSTDMT_JOBSIZE_MAX; + assert(value >= 0); + if (value > ZSTDMT_JOBSIZE_MAX) value = ZSTDMT_JOBSIZE_MAX; params->jobSize = value; return value; - case ZSTDMT_p_overlapSectionLog : - if (value > 9) value = 9; - DEBUGLOG(4, "ZSTDMT_p_overlapSectionLog : %u", value); - params->overlapSizeLog = (value >= 9) ? 9 : value; + + case ZSTDMT_p_overlapLog : + DEBUGLOG(4, "ZSTDMT_p_overlapLog : %i", value); + if (value < ZSTD_OVERLAPLOG_MIN) value = ZSTD_OVERLAPLOG_MIN; + if (value > ZSTD_OVERLAPLOG_MAX) value = ZSTD_OVERLAPLOG_MAX; + params->overlapLog = value; return value; + + case ZSTDMT_p_rsyncable : + value = (value != 0); + params->rsyncable = value; + return value; + default : return ERROR(parameter_unsupported); } } -size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value) +size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value) { DEBUGLOG(4, "ZSTDMT_setMTCtxParameter"); - switch(parameter) - { - case ZSTDMT_p_jobSize : - return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); - case ZSTDMT_p_overlapSectionLog : - return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); - default : - return ERROR(parameter_unsupported); - } + return ZSTDMT_CCtxParam_setMTCtxParameter(&mtctx->params, parameter, value); } -size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value) +size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value) { switch (parameter) { case ZSTDMT_p_jobSize: - *value = mtctx->params.jobSize; + assert(mtctx->params.jobSize <= INT_MAX); + *value = (int)(mtctx->params.jobSize); break; - case ZSTDMT_p_overlapSectionLog: - *value = mtctx->params.overlapSizeLog; + case ZSTDMT_p_overlapLog: + *value = mtctx->params.overlapLog; + break; + case ZSTDMT_p_rsyncable: + *value = mtctx->params.rsyncable; break; default: return ERROR(parameter_unsupported); @@ -1140,22 +1154,66 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx) /* ===== Multi-threaded compression ===== */ /* ------------------------------------------ */ -static size_t ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) +static unsigned ZSTDMT_computeTargetJobLog(ZSTD_CCtx_params const params) { if (params.ldmParams.enableLdm) + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. */ return MAX(21, params.cParams.chainLog + 4); return MAX(20, params.cParams.windowLog + 2); } -static size_t ZSTDMT_computeOverlapLog(ZSTD_CCtx_params const params) +static int ZSTDMT_overlapLog_default(ZSTD_strategy strat) { - unsigned const overlapRLog = (params.overlapSizeLog>9) ? 0 : 9-params.overlapSizeLog; - if (params.ldmParams.enableLdm) - return (MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) - overlapRLog); - return overlapRLog >= 9 ? 0 : (params.cParams.windowLog - overlapRLog); + switch(strat) + { + case ZSTD_btultra2: + return 9; + case ZSTD_btultra: + case ZSTD_btopt: + return 8; + case ZSTD_btlazy2: + case ZSTD_lazy2: + return 7; + case ZSTD_lazy: + case ZSTD_greedy: + case ZSTD_dfast: + case ZSTD_fast: + default:; + } + return 6; } -static unsigned ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) { +static int ZSTDMT_overlapLog(int ovlog, ZSTD_strategy strat) +{ + assert(0 <= ovlog && ovlog <= 9); + if (ovlog == 0) return ZSTDMT_overlapLog_default(strat); + return ovlog; +} + +static size_t ZSTDMT_computeOverlapSize(ZSTD_CCtx_params const params) +{ + int const overlapRLog = 9 - ZSTDMT_overlapLog(params.overlapLog, params.cParams.strategy); + int ovLog = (overlapRLog >= 8) ? 0 : (params.cParams.windowLog - overlapRLog); + assert(0 <= overlapRLog && overlapRLog <= 8); + if (params.ldmParams.enableLdm) { + /* In Long Range Mode, the windowLog is typically oversized. + * In which case, it's preferable to determine the jobSize + * based on chainLog instead. + * Then, ovLog becomes a fraction of the jobSize, rather than windowSize */ + ovLog = MIN(params.cParams.windowLog, ZSTDMT_computeTargetJobLog(params) - 2) + - overlapRLog; + } + assert(0 <= ovLog && ovLog <= 30); + DEBUGLOG(4, "overlapLog : %i", params.overlapLog); + DEBUGLOG(4, "overlap size : %i", 1 << ovLog); + return (ovLog==0) ? 0 : (size_t)1 << ovLog; +} + +static unsigned +ZSTDMT_computeNbJobs(ZSTD_CCtx_params params, size_t srcSize, unsigned nbWorkers) +{ assert(nbWorkers>0); { size_t const jobSizeTarget = (size_t)1 << ZSTDMT_computeTargetJobLog(params); size_t const jobMaxSize = jobSizeTarget << 2; @@ -1178,7 +1236,7 @@ static size_t ZSTDMT_compress_advanced_internal( ZSTD_CCtx_params params) { ZSTD_CCtx_params const jobParams = ZSTDMT_initJobCCtxParams(params); - size_t const overlapSize = (size_t)1 << ZSTDMT_computeOverlapLog(params); + size_t const overlapSize = ZSTDMT_computeOverlapSize(params); unsigned const nbJobs = ZSTDMT_computeNbJobs(params, srcSize, params.nbWorkers); size_t const proposedJobSize = (srcSize + (nbJobs-1)) / nbJobs; size_t const avgJobSize = (((proposedJobSize-1) & 0x1FFFF) < 0x7FFF) ? proposedJobSize + 0xFFFF : proposedJobSize; /* avoid too small last block */ @@ -1289,16 +1347,17 @@ static size_t ZSTDMT_compress_advanced_internal( } size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const ZSTD_CDict* cdict, - ZSTD_parameters params, - unsigned overlapLog) + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_parameters params, + int overlapLog) { ZSTD_CCtx_params cctxParams = mtctx->params; cctxParams.cParams = params.cParams; cctxParams.fParams = params.fParams; - cctxParams.overlapSizeLog = overlapLog; + assert(ZSTD_OVERLAPLOG_MIN <= overlapLog && overlapLog <= ZSTD_OVERLAPLOG_MAX); + cctxParams.overlapLog = overlapLog; return ZSTDMT_compress_advanced_internal(mtctx, dst, dstCapacity, src, srcSize, @@ -1311,8 +1370,8 @@ size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, const void* src, size_t srcSize, int compressionLevel) { - U32 const overlapLog = (compressionLevel >= ZSTD_maxCLevel()) ? 9 : ZSTDMT_OVERLAPLOG_DEFAULT; ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, 0); + int const overlapLog = ZSTDMT_overlapLog_default(params.cParams.strategy); params.fParams.contentSizeFlag = 1; return ZSTDMT_compress_advanced(mtctx, dst, dstCapacity, src, srcSize, NULL, params, overlapLog); } @@ -1339,8 +1398,8 @@ size_t ZSTDMT_initCStream_internal( if (params.nbWorkers != mtctx->params.nbWorkers) CHECK_F( ZSTDMT_resize(mtctx, params.nbWorkers) ); - if (params.jobSize > 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; - if (params.jobSize > ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; + if (params.jobSize != 0 && params.jobSize < ZSTDMT_JOBSIZE_MIN) params.jobSize = ZSTDMT_JOBSIZE_MIN; + if (params.jobSize > (size_t)ZSTDMT_JOBSIZE_MAX) params.jobSize = ZSTDMT_JOBSIZE_MAX; mtctx->singleBlockingThread = (pledgedSrcSize <= ZSTDMT_JOBSIZE_MIN); /* do not trigger multi-threading when srcSize is too small */ if (mtctx->singleBlockingThread) { @@ -1375,14 +1434,24 @@ size_t ZSTDMT_initCStream_internal( mtctx->cdict = cdict; } - mtctx->targetPrefixSize = (size_t)1 << ZSTDMT_computeOverlapLog(params); - DEBUGLOG(4, "overlapLog=%u => %u KB", params.overlapSizeLog, (U32)(mtctx->targetPrefixSize>>10)); + mtctx->targetPrefixSize = ZSTDMT_computeOverlapSize(params); + DEBUGLOG(4, "overlapLog=%i => %u KB", params.overlapLog, (U32)(mtctx->targetPrefixSize>>10)); mtctx->targetSectionSize = params.jobSize; if (mtctx->targetSectionSize == 0) { mtctx->targetSectionSize = 1ULL << ZSTDMT_computeTargetJobLog(params); } + if (params.rsyncable) { + /* Aim for the targetsectionSize as the average job size. */ + U32 const jobSizeMB = (U32)(mtctx->targetSectionSize >> 20); + U32 const rsyncBits = ZSTD_highbit32(jobSizeMB) + 20; + assert(jobSizeMB >= 1); + DEBUGLOG(4, "rsyncLog = %u", rsyncBits); + mtctx->rsync.hash = 0; + mtctx->rsync.hitMask = (1ULL << rsyncBits) - 1; + mtctx->rsync.primePower = ZSTD_rollingHash_primePower(RSYNC_LENGTH); + } if (mtctx->targetSectionSize < mtctx->targetPrefixSize) mtctx->targetSectionSize = mtctx->targetPrefixSize; /* job size must be >= overlap size */ - DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), params.jobSize); + DEBUGLOG(4, "Job Size : %u KB (note : set to %u)", (U32)(mtctx->targetSectionSize>>10), (U32)params.jobSize); DEBUGLOG(4, "inBuff Size : %u KB", (U32)(mtctx->targetSectionSize>>10)); ZSTDMT_setBufferSize(mtctx->bufPool, ZSTD_compressBound(mtctx->targetSectionSize)); { @@ -1818,6 +1887,89 @@ static int ZSTDMT_tryGetInputRange(ZSTDMT_CCtx* mtctx) return 1; } +typedef struct { + size_t toLoad; /* The number of bytes to load from the input. */ + int flush; /* Boolean declaring if we must flush because we found a synchronization point. */ +} syncPoint_t; + +/** + * Searches through the input for a synchronization point. If one is found, we + * will instruct the caller to flush, and return the number of bytes to load. + * Otherwise, we will load as many bytes as possible and instruct the caller + * to continue as normal. + */ +static syncPoint_t +findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) +{ + BYTE const* const istart = (BYTE const*)input.src + input.pos; + U64 const primePower = mtctx->rsync.primePower; + U64 const hitMask = mtctx->rsync.hitMask; + + syncPoint_t syncPoint; + U64 hash; + BYTE const* prev; + size_t pos; + + syncPoint.toLoad = MIN(input.size - input.pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint.flush = 0; + if (!mtctx->params.rsyncable) + /* Rsync is disabled. */ + return syncPoint; + if (mtctx->inBuff.filled + syncPoint.toLoad < RSYNC_LENGTH) + /* Not enough to compute the hash. + * We will miss any synchronization points in this RSYNC_LENGTH byte + * window. However, since it depends only in the internal buffers, if the + * state is already synchronized, we will remain synchronized. + * Additionally, the probability that we miss a synchronization point is + * low: RSYNC_LENGTH / targetSectionSize. + */ + return syncPoint; + /* Initialize the loop variables. */ + if (mtctx->inBuff.filled >= RSYNC_LENGTH) { + /* We have enough bytes buffered to initialize the hash. + * Start scanning at the beginning of the input. + */ + pos = 0; + prev = (BYTE const*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled - RSYNC_LENGTH; + hash = ZSTD_rollingHash_compute(prev, RSYNC_LENGTH); + } else { + /* We don't have enough bytes buffered to initialize the hash, but + * we know we have at least RSYNC_LENGTH bytes total. + * Start scanning after the first RSYNC_LENGTH bytes less the bytes + * already buffered. + */ + pos = RSYNC_LENGTH - mtctx->inBuff.filled; + prev = (BYTE const*)mtctx->inBuff.buffer.start - pos; + hash = ZSTD_rollingHash_compute(mtctx->inBuff.buffer.start, mtctx->inBuff.filled); + hash = ZSTD_rollingHash_append(hash, istart, pos); + } + /* Starting with the hash of the previous RSYNC_LENGTH bytes, roll + * through the input. If we hit a synchronization point, then cut the + * job off, and tell the compressor to flush the job. Otherwise, load + * all the bytes and continue as normal. + * If we go too long without a synchronization point (targetSectionSize) + * then a block will be emitted anyways, but this is okay, since if we + * are already synchronized we will remain synchronized. + */ + for (; pos < syncPoint.toLoad; ++pos) { + BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; + /* if (pos >= RSYNC_LENGTH) assert(ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); */ + hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); + if ((hash & hitMask) == hitMask) { + syncPoint.toLoad = pos + 1; + syncPoint.flush = 1; + break; + } + } + return syncPoint; +} + +size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx) +{ + size_t hintInSize = mtctx->targetSectionSize - mtctx->inBuff.filled; + if (hintInSize==0) hintInSize = mtctx->targetSectionSize; + return hintInSize; +} /** ZSTDMT_compressStream_generic() : * internal use only - exposed to be invoked from zstd_compress.c @@ -1844,7 +1996,8 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, } /* single-pass shortcut (note : synchronous-mode) */ - if ( (mtctx->nextJobID == 0) /* just started */ + if ( (!mtctx->params.rsyncable) /* rsyncable mode is disabled */ + && (mtctx->nextJobID == 0) /* just started */ && (mtctx->inBuff.filled == 0) /* nothing buffered */ && (!mtctx->jobReady) /* no job already created */ && (endOp == ZSTD_e_end) /* end order */ @@ -1876,14 +2029,17 @@ size_t ZSTDMT_compressStream_generic(ZSTDMT_CCtx* mtctx, DEBUGLOG(5, "ZSTDMT_tryGetInputRange completed successfully : mtctx->inBuff.buffer.start = %p", mtctx->inBuff.buffer.start); } if (mtctx->inBuff.buffer.start != NULL) { - size_t const toLoad = MIN(input->size - input->pos, mtctx->targetSectionSize - mtctx->inBuff.filled); + syncPoint_t const syncPoint = findSynchronizationPoint(mtctx, *input); + if (syncPoint.flush && endOp == ZSTD_e_continue) { + endOp = ZSTD_e_flush; + } assert(mtctx->inBuff.buffer.capacity >= mtctx->targetSectionSize); DEBUGLOG(5, "ZSTDMT_compressStream_generic: adding %u bytes on top of %u to buffer of size %u", - (U32)toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); - memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, toLoad); - input->pos += toLoad; - mtctx->inBuff.filled += toLoad; - forwardInputProgress = toLoad>0; + (U32)syncPoint.toLoad, (U32)mtctx->inBuff.filled, (U32)mtctx->targetSectionSize); + memcpy((char*)mtctx->inBuff.buffer.start + mtctx->inBuff.filled, (const char*)input->src + input->pos, syncPoint.toLoad); + input->pos += syncPoint.toLoad; + mtctx->inBuff.filled += syncPoint.toLoad; + forwardInputProgress = syncPoint.toLoad>0; } if ((input->pos < input->size) && (endOp == ZSTD_e_end)) endOp = ZSTD_e_flush; /* can't end now : not all input consumed */ diff --git a/C/zstd/zstdmt_compress.h b/C/zstd/zstdmt_compress.h index 12ad9f89..ee771681 100644 --- a/C/zstd/zstdmt_compress.h +++ b/C/zstd/zstdmt_compress.h @@ -28,6 +28,16 @@ #include "zstd.h" /* ZSTD_inBuffer, ZSTD_outBuffer, ZSTDLIB_API */ +/* === Constants === */ +#ifndef ZSTDMT_NBWORKERS_MAX +# define ZSTDMT_NBWORKERS_MAX 200 +#endif +#ifndef ZSTDMT_JOBSIZE_MIN +# define ZSTDMT_JOBSIZE_MIN (1 MB) +#endif +#define ZSTDMT_JOBSIZE_MAX (MEM_32bits() ? (512 MB) : (1024 MB)) + + /* === Memory management === */ typedef struct ZSTDMT_CCtx_s ZSTDMT_CCtx; ZSTDLIB_API ZSTDMT_CCtx* ZSTDMT_createCCtx(unsigned nbWorkers); @@ -52,6 +62,7 @@ ZSTDLIB_API size_t ZSTDMT_compressCCtx(ZSTDMT_CCtx* mtctx, ZSTDLIB_API size_t ZSTDMT_initCStream(ZSTDMT_CCtx* mtctx, int compressionLevel); ZSTDLIB_API size_t ZSTDMT_resetCStream(ZSTDMT_CCtx* mtctx, unsigned long long pledgedSrcSize); /**< if srcSize is not known at reset time, use ZSTD_CONTENTSIZE_UNKNOWN. Note: for compatibility with older programs, 0 means the same as ZSTD_CONTENTSIZE_UNKNOWN, but it will change in the future to mean "empty" */ +ZSTDLIB_API size_t ZSTDMT_nextInputSizeHint(const ZSTDMT_CCtx* mtctx); ZSTDLIB_API size_t ZSTDMT_compressStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTDMT_flushStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /**< @return : 0 == all flushed; >0 : still some data to be flushed; or an error code (ZSTD_isError()) */ @@ -60,16 +71,12 @@ ZSTDLIB_API size_t ZSTDMT_endStream(ZSTDMT_CCtx* mtctx, ZSTD_outBuffer* output); /* === Advanced functions and parameters === */ -#ifndef ZSTDMT_JOBSIZE_MIN -# define ZSTDMT_JOBSIZE_MIN (1U << 20) /* 1 MB - Minimum size of each compression job */ -#endif - ZSTDLIB_API size_t ZSTDMT_compress_advanced(ZSTDMT_CCtx* mtctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const ZSTD_CDict* cdict, ZSTD_parameters params, - unsigned overlapLog); + int overlapLog); ZSTDLIB_API size_t ZSTDMT_initCStream_advanced(ZSTDMT_CCtx* mtctx, const void* dict, size_t dictSize, /* dict can be released after init, a local copy is preserved within zcs */ @@ -84,8 +91,9 @@ ZSTDLIB_API size_t ZSTDMT_initCStream_usingCDict(ZSTDMT_CCtx* mtctx, /* ZSTDMT_parameter : * List of parameters that can be set using ZSTDMT_setMTCtxParameter() */ typedef enum { - ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ - ZSTDMT_p_overlapSectionLog /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ + ZSTDMT_p_jobSize, /* Each job is compressed in parallel. By default, this value is dynamically determined depending on compression parameters. Can be set explicitly here. */ + ZSTDMT_p_overlapLog, /* Each job may reload a part of previous job to enhance compressionr ratio; 0 == no overlap, 6(default) == use 1/8th of window, >=9 == use full window. This is a "sticky" parameter : its value will be re-used on next compression job */ + ZSTDMT_p_rsyncable /* Enables rsyncable mode. */ } ZSTDMT_parameter; /* ZSTDMT_setMTCtxParameter() : @@ -93,12 +101,12 @@ typedef enum { * The function must be called typically after ZSTD_createCCtx() but __before ZSTDMT_init*() !__ * Parameters not explicitly reset by ZSTDMT_init*() remain the same in consecutive compression sessions. * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned value); +ZSTDLIB_API size_t ZSTDMT_setMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int value); /* ZSTDMT_getMTCtxParameter() : * Query the ZSTDMT_CCtx for a parameter value. * @return : 0, or an error code (which can be tested using ZSTD_isError()) */ -ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, unsigned* value); +ZSTDLIB_API size_t ZSTDMT_getMTCtxParameter(ZSTDMT_CCtx* mtctx, ZSTDMT_parameter parameter, int* value); /*! ZSTDMT_compressStream_generic() : @@ -129,7 +137,7 @@ size_t ZSTDMT_toFlushNow(ZSTDMT_CCtx* mtctx); /*! ZSTDMT_CCtxParam_setMTCtxParameter() * like ZSTDMT_setMTCtxParameter(), but into a ZSTD_CCtx_Params */ -size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, unsigned value); +size_t ZSTDMT_CCtxParam_setMTCtxParameter(ZSTD_CCtx_params* params, ZSTDMT_parameter parameter, int value); /*! ZSTDMT_CCtxParam_setNbWorkers() * Set nbWorkers, and clamp it. diff --git a/C/zstdmt/brotli-mt_compress.c b/C/zstdmt/brotli-mt_compress.c index 107d759e..a18b2ac7 100644 --- a/C/zstdmt/brotli-mt_compress.c +++ b/C/zstdmt/brotli-mt_compress.c @@ -267,7 +267,7 @@ static void *pt_compress(void *arg) uint8_t *obuf = (uint8_t*)wl->out.buf + 16; wl->out.size -= 16; rv = BrotliEncoderCompress(ctx->level, - BROTLI_LARGE_MAX_WINDOW_BITS, + BROTLI_MAX_WINDOW_BITS, BROTLI_MODE_GENERIC, in.size, ibuf, &wl->out.size, obuf); diff --git a/C/zstdmt/lz4-mt_compress.c b/C/zstdmt/lz4-mt_compress.c index 0e479619..24ed4929 100644 --- a/C/zstdmt/lz4-mt_compress.c +++ b/C/zstdmt/lz4-mt_compress.c @@ -291,11 +291,13 @@ static void *pt_compress(void *arg) } /* write skippable frame */ - MEM_writeLE32((unsigned char *)wl->out.buf + 0, - LZ4FMT_MAGIC_SKIPPABLE); - MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4); - MEM_writeLE32((unsigned char *)wl->out.buf + 8, (U32) result); - wl->out.size = result + 12; + if (ctx->threads > 1) { + MEM_writeLE32((unsigned char *)wl->out.buf + 0, + LZ4FMT_MAGIC_SKIPPABLE); + MEM_writeLE32((unsigned char *)wl->out.buf + 4, 4); + MEM_writeLE32((unsigned char *)wl->out.buf + 8, (U32) result); + wl->out.size = result + 12; + } /* write result */ pthread_mutex_lock(&ctx->write_mutex); diff --git a/CPP/7zip/Aes.mak b/CPP/7zip/Aes.mak index 4d5e98b3..3d0d8776 100644 --- a/CPP/7zip/Aes.mak +++ b/CPP/7zip/Aes.mak @@ -1,7 +1,7 @@ C_OBJS = $(C_OBJS) \ $O\Aes.obj -!IF "$(CPU)" != "IA64" && "$(CPU)" != "MIPS" && "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" +!IF "$(PLATFORM)" != "ia64" && "$(PLATFORM)" != "mips" && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" ASM_OBJS = $(ASM_OBJS) \ $O\AesOpt.obj !ENDIF diff --git a/CPP/7zip/Archive/7z/7zDecode.cpp b/CPP/7zip/Archive/7z/7zDecode.cpp index 2705ecb9..f61ad4ad 100644 --- a/CPP/7zip/Archive/7z/7zDecode.cpp +++ b/CPP/7zip/Archive/7z/7zDecode.cpp @@ -355,7 +355,9 @@ HRESULT CDecoder::Decode( unsigned i; + #if !defined(_7ZIP_ST) bool mt_wasUsed = false; + #endif for (i = 0; i < folderInfo.Coders.Size(); i++) { diff --git a/CPP/7zip/Archive/7z/7zHandler.cpp b/CPP/7zip/Archive/7z/7zHandler.cpp index 4ff87f76..90565bcd 100644 --- a/CPP/7zip/Archive/7z/7zHandler.cpp +++ b/CPP/7zip/Archive/7z/7zHandler.cpp @@ -236,6 +236,13 @@ STDMETHODIMP CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value) prop = v; break; } + + case kpidReadOnly: + { + if (!_db.CanUpdate()) + prop = true; + break; + } } prop.Detach(value); return S_OK; @@ -507,7 +514,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const else if (id == k_LZ4) { name = "LZ4"; - if (propsSize == 5) + if (propsSize == 3 || propsSize == 5) { char *dest = s; *dest++ = 'v'; @@ -525,7 +532,7 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const else if (id == k_LZ5) { name = "LZ5"; - if (propsSize == 5) + if (propsSize == 3 || propsSize == 5) { char *dest = s; *dest++ = 'v'; @@ -543,9 +550,10 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const else if (id == k_ZSTD) { name = "ZSTD"; - if (propsSize == 5) + if (propsSize == 3 || propsSize == 5) { char *dest = s; + UInt32 l = props[2]; *dest++ = 'v'; ConvertUInt32ToString(props[0], dest); dest += MyStringLen(dest); @@ -553,8 +561,14 @@ HRESULT CHandler::SetMethodToProp(CNum folderIndex, PROPVARIANT *prop) const ConvertUInt32ToString(props[1], dest); dest += MyStringLen(dest); *dest++ = ','; - *dest++ = 'l'; - ConvertUInt32ToString(props[2], dest); + if (l <= 22) { + *dest++ = 'l'; + ConvertUInt32ToString(l, dest); + } else { + *dest++ = 'f'; + *dest++ = 'l'; + ConvertUInt32ToString(l - 32, dest); + } dest += MyStringLen(dest); } } diff --git a/CPP/7zip/Archive/7z/7zHandlerOut.cpp b/CPP/7zip/Archive/7z/7zHandlerOut.cpp index bfdf0943..123143e7 100644 --- a/CPP/7zip/Archive/7z/7zHandlerOut.cpp +++ b/CPP/7zip/Archive/7z/7zHandlerOut.cpp @@ -267,6 +267,9 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt db = &_db; #endif + if (db && !db->CanUpdate()) + return E_NOTIMPL; + /* CMyComPtr getRawProps; updateCallback->QueryInterface(IID_IArchiveGetRawProps, (void **)&getRawProps); diff --git a/CPP/7zip/Archive/7z/7zIn.cpp b/CPP/7zip/Archive/7z/7zIn.cpp index bbc77b03..ae5ff196 100644 --- a/CPP/7zip/Archive/7z/7zIn.cpp +++ b/CPP/7zip/Archive/7z/7zIn.cpp @@ -1465,21 +1465,24 @@ void CDbEx::FillLinks() } if (indexInFolder != 0) + { folderIndex++; - /* - if (indexInFolder != 0) - ThrowIncorrect(); - */ + // 18.06 + ThereIsHeaderError = true; + // ThrowIncorrect(); + } for (;;) { if (folderIndex >= NumFolders) return; FolderStartFileIndex[folderIndex] = i; - /* if (NumUnpackStreamsVector[folderIndex] != 0) - ThrowIncorrect();; - */ + { + // 18.06 + ThereIsHeaderError = true; + // ThrowIncorrect(); + } folderIndex++; } } diff --git a/CPP/7zip/Archive/7z/7zIn.h b/CPP/7zip/Archive/7z/7zIn.h index c5fb9095..bb0e4748 100644 --- a/CPP/7zip/Archive/7z/7zIn.h +++ b/CPP/7zip/Archive/7z/7zIn.h @@ -257,6 +257,16 @@ struct CDbEx: public CDatabase PhySize = 0; } + bool CanUpdate() const + { + if (ThereIsHeaderError + || UnexpectedEnd + || StartHeaderWasRecovered + || UnsupportedFeatureError) + return false; + return true; + } + void FillLinks(); UInt64 GetFolderStreamPos(CNum folderIndex, unsigned indexInFolder) const diff --git a/CPP/7zip/Archive/7z/7zUpdate.cpp b/CPP/7zip/Archive/7z/7zUpdate.cpp index 25a9100d..882ffd6d 100644 --- a/CPP/7zip/Archive/7z/7zUpdate.cpp +++ b/CPP/7zip/Archive/7z/7zUpdate.cpp @@ -124,13 +124,13 @@ static int Parse_EXE(const Byte *buf, size_t size, CFilterMode *filterMode) #define ELF_DATA_2LSB 1 #define ELF_DATA_2MSB 2 -static UInt16 Get16(const Byte *p, Bool be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } -static UInt32 Get32(const Byte *p, Bool be) { if (be) return GetBe32(p); return GetUi32(p); } -// static UInt64 Get64(const Byte *p, Bool be) { if (be) return GetBe64(p); return GetUi64(p); } +static UInt16 Get16(const Byte *p, BoolInt be) { if (be) return (UInt16)GetBe16(p); return (UInt16)GetUi16(p); } +static UInt32 Get32(const Byte *p, BoolInt be) { if (be) return GetBe32(p); return GetUi32(p); } +// static UInt64 Get64(const Byte *p, BoolInt be) { if (be) return GetBe64(p); return GetUi64(p); } static int Parse_ELF(const Byte *buf, size_t size, CFilterMode *filterMode) { - Bool /* is32, */ be; + BoolInt /* is32, */ be; UInt32 filterId; if (size < 512 || buf[6] != 1) /* ver */ @@ -200,7 +200,7 @@ static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode if (size < 512) return 0; - Bool /* mode64, */ be; + BoolInt /* mode64, */ be; switch (GetUi32(buf)) { case MACH_SIG_BE_32: /* mode64 = False; */ be = True; break; @@ -239,7 +239,7 @@ static unsigned Parse_MACH(const Byte *buf, size_t size, CFilterMode *filterMode #define RIFF_SIG 0x46464952 -static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) +static BoolInt Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) { UInt32 subChunkSize, pos; if (size < 0x2C) @@ -285,7 +285,7 @@ static Bool Parse_WAV(const Byte *buf, size_t size, CFilterMode *filterMode) return False; } -static Bool ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) +static BoolInt ParseFile(const Byte *buf, size_t size, CFilterMode *filterMode) { filterMode->Id = 0; filterMode->Delta = 0; @@ -894,7 +894,7 @@ HRESULT CAnalysis::GetFilterGroup(UInt32 index, const CUpdateItem &ui, CFilterMo // RINOK(Callback->SetOperationResult2(index, NUpdate::NOperationResult::kOK)); if (result == S_OK) { - Bool parseRes = ParseFile(Buffer, size, &filterModeTemp); + BoolInt parseRes = ParseFile(Buffer, size, &filterModeTemp); if (parseRes && filterModeTemp.Delta == 0) { filterModeTemp.SetDelta(); @@ -1648,6 +1648,9 @@ HRESULT Update( for (CNum fi = db->FolderStartFileIndex[i]; indexInFolder < numUnpackStreams; fi++) { + if (fi >= db->Files.Size()) + return E_FAIL; + const CFileItem &file = db->Files[fi]; if (file.HasStream) { diff --git a/CPP/7zip/Archive/ArHandler.cpp b/CPP/7zip/Archive/ArHandler.cpp index 32bbb341..85feeeab 100644 --- a/CPP/7zip/Archive/ArHandler.cpp +++ b/CPP/7zip/Archive/ArHandler.cpp @@ -170,8 +170,8 @@ static bool OctalToNumber32(const char *s, unsigned size, UInt32 &res) res = 0; char sz[32]; size = RemoveTailSpaces(sz, s, size); - if (size == 0) - return true; // some items doesn't contaion any numbers + if (size == 0 || strcmp(sz, "-1") == 0) + return true; // some items don't contain any numbers const char *end; UInt64 res64 = ConvertOctStringToUInt64(sz, &end); if ((unsigned)(end - sz) != size) @@ -185,8 +185,8 @@ static bool DecimalToNumber(const char *s, unsigned size, UInt64 &res) res = 0; char sz[32]; size = RemoveTailSpaces(sz, s, size); - if (size == 0) - return true; // some items doesn't contaion any numbers + if (size == 0 || strcmp(sz, "-1") == 0) + return true; // some items don't contain any numbers const char *end; res = ConvertStringToUInt64(sz, &end); return ((unsigned)(end - sz) == size); diff --git a/CPP/7zip/Archive/Bz2Handler.cpp b/CPP/7zip/Archive/Bz2Handler.cpp index 6377b638..d12ccc8f 100644 --- a/CPP/7zip/Archive/Bz2Handler.cpp +++ b/CPP/7zip/Archive/Bz2Handler.cpp @@ -366,7 +366,13 @@ STDMETHODIMP CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numIt return E_INVALIDARG; size = prop.uhVal.QuadPart; } - return UpdateArchive(size, outStream, _props, updateCallback); + + CMethodProps props2 = _props; + #ifndef _7ZIP_ST + props2.AddProp_NumThreads(_props._numThreads); + #endif + + return UpdateArchive(size, outStream, props2, updateCallback); } if (indexInArchive != 0) diff --git a/CPP/7zip/Archive/Common/CoderMixer2.cpp b/CPP/7zip/Archive/Common/CoderMixer2.cpp index d4d99495..baddddfc 100644 --- a/CPP/7zip/Archive/Common/CoderMixer2.cpp +++ b/CPP/7zip/Archive/Common/CoderMixer2.cpp @@ -91,6 +91,7 @@ HRESULT CCoder::CheckDataAfterEnd(bool &dataAfterEnd_Error /* , bool &InternalPa { CMyComPtr getInStreamProcessedSize2; Coder2.QueryInterface(IID_ICompressGetInStreamProcessedSize2, (void **)&getInStreamProcessedSize2); + if (getInStreamProcessedSize2) FOR_VECTOR (i, PackSizePointers) { if (!PackSizePointers[i]) diff --git a/CPP/7zip/Archive/DmgHandler.cpp b/CPP/7zip/Archive/DmgHandler.cpp index 5d7415ee..88238e95 100644 --- a/CPP/7zip/Archive/DmgHandler.cpp +++ b/CPP/7zip/Archive/DmgHandler.cpp @@ -788,9 +788,21 @@ HRESULT CHandler::Open2(IInStream *stream) if (headSize != RSRC_HEAD_SIZE || footerOffset >= rsrcPair.Len || mainDataSize >= rsrcPair.Len - || footerOffset + footerSize != rsrcPair.Len + || footerOffset < mainDataSize || footerOffset != headSize + mainDataSize) return S_FALSE; + + const UInt32 footerEnd = footerOffset + footerSize; + if (footerEnd != rsrcPair.Len) + { + // there is rare case dmg example, where there are 4 additional bytes + UInt64 rem = rsrcPair.Len - footerOffset; + if (rem < footerSize + || rem - footerSize != 4 + || Get32(p + footerEnd) != 0) + return S_FALSE; + } + if (footerSize < 16) return S_FALSE; if (memcmp(p, p + footerOffset, 16) != 0) diff --git a/CPP/7zip/Archive/ExtHandler.cpp b/CPP/7zip/Archive/ExtHandler.cpp index 0d46cd63..07be3a69 100644 --- a/CPP/7zip/Archive/ExtHandler.cpp +++ b/CPP/7zip/Archive/ExtHandler.cpp @@ -1596,6 +1596,17 @@ STDMETHODIMP CHandler::Close() } +static void ChangeSeparatorsInName(char *s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + { + char c = s[i]; + if (c == CHAR_PATH_SEPARATOR || c == '/') + s[i] = '_'; + } +} + + void CHandler::GetPath(unsigned index, AString &s) const { s.Empty(); @@ -1612,6 +1623,8 @@ void CHandler::GetPath(unsigned index, AString &s) const if (!s.IsEmpty()) s.InsertAtFront(CHAR_PATH_SEPARATOR); s.Insert(0, item.Name); + // 18.06 + ChangeSeparatorsInName(s.GetBuf(), item.Name.Len()); if (item.ParentNode == k_INODE_ROOT) return; diff --git a/CPP/7zip/Archive/FatHandler.cpp b/CPP/7zip/Archive/FatHandler.cpp index c97e46e2..eccf1aa1 100644 --- a/CPP/7zip/Archive/FatHandler.cpp +++ b/CPP/7zip/Archive/FatHandler.cpp @@ -161,7 +161,8 @@ bool CHeader::Parse(const Byte *p) return false; // we also support images that contain 0 in offset field. - bool isOkOffset = (codeOffset == 0 || (p[0] == 0xEB && p[1] == 0)); + bool isOkOffset = (codeOffset == 0) + || (codeOffset == (p[0] == 0xEB ? 2 : 3)); UInt16 numRootDirEntries = Get16(p + 17); if (numRootDirEntries == 0) @@ -732,7 +733,13 @@ HRESULT CDatabase::Open() RINOK(OpenProgressFat()); if ((Fat[0] & 0xFF) != Header.MediaType) - return S_FALSE; + { + // that case can mean error in FAT, + // but xdf file: (MediaType == 0xF0 && Fat[0] == 0xFF9) + // 19.00: so we use non-strict check + if ((Fat[0] & 0xFF) < 0xF0) + return S_FALSE; + } RINOK(ReadDir(-1, Header.RootCluster, 0)); diff --git a/CPP/7zip/Archive/HfsHandler.cpp b/CPP/7zip/Archive/HfsHandler.cpp index 83a0d413..5ccbae63 100644 --- a/CPP/7zip/Archive/HfsHandler.cpp +++ b/CPP/7zip/Archive/HfsHandler.cpp @@ -419,7 +419,13 @@ void CDatabase::GetItemPath(unsigned index, NWindows::NCOM::CPropVariant &path) const wchar_t *src = (const wchar_t *)*s; wchar_t *dest = p + len; for (unsigned j = 0; j < curLen; j++) - dest[j] = src[j]; + { + wchar_t c = src[j]; + // 18.06 + if (c == CHAR_PATH_SEPARATOR || c == '/') + c = '_'; + dest[j] = c; + } if (len == 0) break; diff --git a/CPP/7zip/Archive/LzmaHandler.cpp b/CPP/7zip/Archive/LzmaHandler.cpp index 28079dfc..f13fca7a 100644 --- a/CPP/7zip/Archive/LzmaHandler.cpp +++ b/CPP/7zip/Archive/LzmaHandler.cpp @@ -131,13 +131,7 @@ HRESULT CDecoder::Code(const CHeader &header, ISequentialOutStream *outStream, if (header.FilterID > 1) return E_NOTIMPL; - { - CMyComPtr setDecoderProperties; - _lzmaDecoder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecoderProperties); - if (!setDecoderProperties) - return E_NOTIMPL; - RINOK(setDecoderProperties->SetDecoderProperties2(header.LzmaProps, 5)); - } + RINOK(_lzmaDecoderSpec->SetDecoderProperties2(header.LzmaProps, 5)); bool filteredMode = (header.FilterID == 1); @@ -357,24 +351,54 @@ API_FUNC_static_IsArc IsArc_Lzma86(const Byte *p, size_t size) } } + + STDMETHODIMP CHandler::Open(IInStream *inStream, const UInt64 *, IArchiveOpenCallback *) { Close(); - const UInt32 kBufSize = 1 + 5 + 8 + 2; + const unsigned headerSize = GetHeaderSize(); + const UInt32 kBufSize = 1 << 7; Byte buf[kBufSize]; - - RINOK(ReadStream_FALSE(inStream, buf, kBufSize)); - + size_t processedSize = kBufSize; + RINOK(ReadStream(inStream, buf, &processedSize)); + if (processedSize < headerSize + 2) + return S_FALSE; if (!_header.Parse(buf, _lzma86)) return S_FALSE; - const Byte *start = buf + GetHeaderSize(); + const Byte *start = buf + headerSize; if (start[0] != 0 /* || (start[1] & 0x80) != 0 */ ) // empty stream with EOS is not 0x80 return S_FALSE; - + RINOK(inStream->Seek(0, STREAM_SEEK_END, &_packSize)); - if (_packSize >= 24 && _header.Size == 0 && _header.FilterID == 0 && _header.LzmaProps[0] == 0) + + SizeT srcLen = processedSize - headerSize; + + if (srcLen > 10 + && _header.Size == 0 + // && _header.FilterID == 0 + && _header.LzmaProps[0] == 0 + ) return S_FALSE; + + CDecoder state; + const UInt32 outLimit = 1 << 11; + Byte outBuf[outLimit]; + + SizeT outSize = outLimit; + if (outSize > _header.Size) + outSize = (SizeT)_header.Size; + SizeT destLen = outSize; + ELzmaStatus status; + + SRes res = LzmaDecode(outBuf, &destLen, start, &srcLen, + _header.LzmaProps, 5, LZMA_FINISH_ANY, + &status, &g_Alloc); + + if (res != SZ_OK) + if (res != SZ_ERROR_INPUT_EOF) + return S_FALSE; + _isArc = true; _stream = inStream; _seqStream = inStream; diff --git a/CPP/7zip/Archive/Nsis/NsisIn.cpp b/CPP/7zip/Archive/Nsis/NsisIn.cpp index 64463ceb..62eb2264 100644 --- a/CPP/7zip/Archive/Nsis/NsisIn.cpp +++ b/CPP/7zip/Archive/Nsis/NsisIn.cpp @@ -2445,23 +2445,24 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) bool strongPark = false; bool strongNsis = false; + if (NumStringChars > 2) { const Byte *strData = _data + _stringsPos; if (IsUnicode) { - UInt32 num = NumStringChars; + UInt32 num = NumStringChars - 2; for (UInt32 i = 0; i < num; i++) { if (Get16(strData + i * 2) == 0) { unsigned c2 = Get16(strData + 2 + i * 2); + // it can be TXT/RTF with marker char (1 or 2). so we must check next char // if (c2 <= NS_3_CODE_SKIP && c2 != NS_3_CODE_SHELL) if (c2 == NS_3_CODE_VAR) { - // it can be TXT/RTF string with marker char (1 or 2). so we must next char - // const wchar_t *p2 = (const wchar_t *)(strData + i * 2 + 2); - // p2 = p2; - if ((Get16(strData + 3 + i * 2) & 0x8000) != 0) + // 18.06: fixed: is it correct ? + // if ((Get16(strData + 3 + i * 2) & 0x8000) != 0) + if ((Get16(strData + 4 + i * 2) & 0x8080) == 0x8080) { NsisType = k_NsisType_Nsis3; strongNsis = true; @@ -2478,7 +2479,7 @@ void CInArchive::DetectNsisType(const CBlockHeader &bh, const Byte *p) } else { - UInt32 num = NumStringChars; + UInt32 num = NumStringChars - 2; for (UInt32 i = 0; i < num; i++) { if (strData[i] == 0) diff --git a/CPP/7zip/Archive/NtfsHandler.cpp b/CPP/7zip/Archive/NtfsHandler.cpp index 288da156..3e037b13 100644 --- a/CPP/7zip/Archive/NtfsHandler.cpp +++ b/CPP/7zip/Archive/NtfsHandler.cpp @@ -1475,6 +1475,21 @@ void CDatabase::ClearAndClose() InStream.Release(); } + +static void CopyName(wchar_t *dest, const wchar_t *src) +{ + for (;;) + { + wchar_t c = *src++; + // 18.06 + if (c == '\\' || c == '/') + c = '_'; + *dest++ = c; + if (c == 0) + return; + } +} + void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const { const CItem *item = &Items[index]; @@ -1492,7 +1507,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const wchar_t *s = path.AllocBstr(data.Name.Len() + 1); s[0] = L':'; if (!data.Name.IsEmpty()) - MyStringCopy(s + 1, data.Name.GetRawPtr()); + CopyName(s + 1, data.Name.GetRawPtr()); return; } @@ -1541,7 +1556,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const if (!name.IsEmpty()) { size -= name.Len(); - MyStringCopy(s + size, name.GetRawPtr()); + CopyName(s + size, name.GetRawPtr()); } s[--size] = ':'; needColon = true; @@ -1551,7 +1566,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const const UString2 &name = rec.FileNames[item->NameIndex].Name; unsigned len = name.Len(); if (len != 0) - MyStringCopy(s + size - len, name.GetRawPtr()); + CopyName(s + size - len, name.GetRawPtr()); if (needColon) s[size] = ':'; size -= len; @@ -1575,7 +1590,7 @@ void CDatabase::GetItemPath(unsigned index, NCOM::CPropVariant &path) const if (len != 0) { size -= len; - MyStringCopy(s + size, name.GetRawPtr()); + CopyName(s + size, name.GetRawPtr()); } s[size + len] = WCHAR_PATH_SEPARATOR; continue; diff --git a/CPP/7zip/Archive/PeHandler.cpp b/CPP/7zip/Archive/PeHandler.cpp index 2457009a..ca833cf8 100644 --- a/CPP/7zip/Archive/PeHandler.cpp +++ b/CPP/7zip/Archive/PeHandler.cpp @@ -41,9 +41,8 @@ static const UInt32 k_Signature32 = 0x00004550; static HRESULT CalcCheckSum(ISequentialInStream *stream, UInt32 size, UInt32 excludePos, UInt32 &res) { - const UInt32 kBufSizeMax = (UInt32)1 << 16; - UInt32 bufSize = MyMin(kBufSizeMax, size); - bufSize += (bufSize & 1); + const UInt32 kBufSizeMax = (UInt32)1 << 15; + UInt32 bufSize = kBufSizeMax; CByteBuffer buffer(bufSize); Byte *buf = buffer; UInt32 sum = 0; @@ -58,9 +57,6 @@ static HRESULT CalcCheckSum(ISequentialInStream *stream, UInt32 size, UInt32 exc size_t processed = rem; RINOK(ReadStream(stream, buf, &processed)); - if ((processed & 1) != 0) - buf[processed] = 0; - for (unsigned j = 0; j < 4; j++) { UInt32 e = excludePos + j; @@ -72,11 +68,30 @@ static HRESULT CalcCheckSum(ISequentialInStream *stream, UInt32 size, UInt32 exc } } - for (size_t i = 0; i < processed; i += 2) + const unsigned kStep = (1 << 4); { - sum += Get16(buf + i); - sum = (sum + (sum >> 16)) & 0xFFFF; + for (size_t i = processed; (i & (kStep - 1)) != 0; i++) + buf[i] = 0; } + { + const Byte *buf2 = buf; + const Byte *bufLimit = buf + processed; + UInt64 sum2 = 0; + for (; buf2 < bufLimit; buf2 += kStep) + { + UInt64 sum3 = (UInt64)Get32(buf2) + + Get32(buf2 + 4) + + Get32(buf2 + 8) + + Get32(buf2 + 12); + sum2 += sum3; + } + sum2 = (UInt32)(sum2) + (UInt64)(sum2 >> 32); + UInt32 sum3 = ((UInt32)sum2 + (UInt32)(sum2 >> 32)); + sum += (sum3 & 0xFFFF) + (sum3 >> 16); + sum = (sum & 0xFFFF) + (sum >> 16); + sum = (sum & 0xFFFF) + (sum >> 16); + } + pos += (UInt32)processed; if (rem != processed) break; diff --git a/CPP/7zip/Archive/Rar/Rar5Handler.cpp b/CPP/7zip/Archive/Rar/Rar5Handler.cpp index 61c9fc72..96238239 100644 --- a/CPP/7zip/Archive/Rar/Rar5Handler.cpp +++ b/CPP/7zip/Archive/Rar/Rar5Handler.cpp @@ -7,6 +7,7 @@ #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/UTFConvert.h" #include "../../../Windows/PropVariantUtils.h" @@ -104,37 +105,6 @@ static const char * const g_LinkTypes[] = static const char g_ExtraTimeFlags[] = { 'u', 'M', 'C', 'A', 'n' }; -template -struct CAlignedBuffer -{ - Byte *_buf; - Byte *_bufBase; - size_t _size; - - CAlignedBuffer(): _buf(NULL), _bufBase(NULL), _size(0) {} - ~CAlignedBuffer() { ::MyFree(_bufBase); } -public: - operator Byte *() { return _buf; } - operator const Byte *() const { return _buf; } - - void AllocAtLeast(size_t size) - { - if (_buf && _size >= size) - return; - ::MyFree(_bufBase); - _buf = NULL; - _size = 0; - _bufBase = (Byte *)::MyAlloc(size + alignMask); - - if (_bufBase) - { - _size = size; - // _buf = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - _buf = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - } - } -}; - static unsigned ReadVarInt(const Byte *p, size_t maxSize, UInt64 *val) { *val = 0; @@ -578,7 +548,7 @@ STDMETHODIMP COutStreamWithHash::Write(const void *data, UInt32 size, UInt32 *pr class CInArchive { - CAlignedBuffer _buf; + CAlignedBuffer _buf; size_t _bufSize; size_t _bufPos; ISequentialInStream *_stream; @@ -586,7 +556,7 @@ class CInArchive NCrypto::NRar5::CDecoder *m_CryptoDecoderSpec; CMyComPtr m_CryptoDecoder; - + CLASS_NO_COPY(CInArchive) HRESULT ReadStream_Check(void *data, size_t size); @@ -610,6 +580,8 @@ public: UInt64 DataSize; }; + CInArchive() {} + HRESULT ReadBlockHeader(CHeader &h); bool ReadFileHeader(const CHeader &header, CItem &item); void AddToSeekValue(UInt64 addValue) @@ -1000,7 +972,7 @@ bool CInArchive::ReadFileHeader(const CHeader &header, CItem &item) struct CLinkFile { unsigned Index; - unsigned NumLinks; + unsigned NumLinks; // the number of links to Data CByteBuffer Data; HRESULT Res; bool crcOK; @@ -1015,7 +987,7 @@ struct CUnpacker CMyComPtr copyCoder; CMyComPtr LzCoders[2]; - bool NeedClearSolid[2]; + bool SolidAllowed; CFilterCoder *filterStreamSpec; CMyComPtr filterStream; @@ -1032,7 +1004,7 @@ struct CUnpacker CLinkFile *linkFile; - CUnpacker(): linkFile(NULL) { NeedClearSolid[0] = NeedClearSolid[1] = true; } + CUnpacker(): linkFile(NULL) { SolidAllowed = false; } HRESULT Create(DECL_EXTERNAL_CODECS_LOC_VARS const CItem &item, bool isSolid, bool &wrongPassword); @@ -1142,7 +1114,9 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz if (method > kLzMethodMax) return E_NOTIMPL; - if (linkFile && !lastItem.Is_UnknownSize()) + bool needBuf = (linkFile && linkFile->NumLinks != 0); + + if (needBuf && !lastItem.Is_UnknownSize()) { size_t dataSize = (size_t)lastItem.Size; if (dataSize != lastItem.Size) @@ -1167,15 +1141,15 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz ICompressCoder *commonCoder = (method == 0) ? copyCoder : LzCoders[item.IsService() ? 1 : 0]; outStreamSpec->SetStream(realOutStream); - outStreamSpec->Init(lastItem, (linkFile ? (Byte *)linkFile->Data : NULL)); - - NeedClearSolid[item.IsService() ? 1 : 0] = false; + outStreamSpec->Init(lastItem, (needBuf ? (Byte *)linkFile->Data : NULL)); HRESULT res = S_OK; if (packSize != 0 || lastItem.Is_UnknownSize() || lastItem.Size != 0) { res = commonCoder->Code(inStream, outStream, &packSize, lastItem.Is_UnknownSize() ? NULL : &lastItem.Size, progress); + if (!item.IsService()) + SolidAllowed = true; } else { @@ -1210,7 +1184,9 @@ HRESULT CUnpacker::Code(const CItem &item, const CItem &lastItem, UInt64 packSiz { linkFile->Res = res; linkFile->crcOK = isCrcOK; - if (!lastItem.Is_UnknownSize() && processedSize != lastItem.Size) + if (needBuf + && !lastItem.Is_UnknownSize() + && processedSize != lastItem.Size) linkFile->Data.ChangeSize_KeepData((size_t)processedSize, (size_t)processedSize); } @@ -2578,6 +2554,15 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, const Byte kStatus_Skip = 1 << 1; const Byte kStatus_Link = 1 << 2; + /* + In original RAR: + 1) service streams are not allowed to be solid, + and solid flag must be ignored for service streams. + 2) If RAR creates new solid block and first file in solid block is Link file, + then it can clear solid flag for Link file and + clear solid flag for first non-Link file after Link file. + */ + CObjectVector linkFiles; { @@ -2603,13 +2588,15 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (ref.Link >= 0) { - if (!testMode) + // 18.06 fixed: we use links for Test mode too + // if (!testMode) { if ((unsigned)ref.Link < index) { const CRefItem &linkRef = _refs[(unsigned)ref.Link]; const CItem &linkItem = _items[linkRef.Item]; - if (linkItem.IsSolid() && linkItem.Size <= k_CopyLinkFile_MaxSize) + if (linkItem.IsSolid()) + if (testMode || linkItem.Size <= k_CopyLinkFile_MaxSize) { if (extractStatuses[(unsigned)ref.Link] == 0) { @@ -2664,19 +2651,25 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, { unsigned solidLimit = 0; - FOR_VECTOR(i, _refs) + FOR_VECTOR (i, _refs) { if ((extractStatuses[i] & kStatus_Link) == 0) continue; + + // We use CLinkFile for testMode too. + // So we can show errors for copy files. + // if (!testMode) + { + CLinkFile &linkFile = linkFiles.AddNew(); + linkFile.Index = i; + } + const CItem &item = _items[_refs[i].Item]; /* if (item.IsService()) continue; */ - CLinkFile &linkFile = linkFiles.AddNew(); - linkFile.Index = i; - if (item.IsSolid()) { unsigned j = i; @@ -2707,6 +2700,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, solidLimit = i + 1; } + if (!testMode) for (UInt32 t = 0; t < numItems; t++) { unsigned index = allFilesMode ? t : indices[t]; @@ -2748,7 +2742,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, // bool needClearSolid = true; - FOR_VECTOR(i, _refs) + FOR_VECTOR (i, _refs) { if (extractStatuses[i] == 0) continue; @@ -2761,15 +2755,19 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, CMyComPtr realOutStream; + // isExtract means that we don't skip that item. So we need read data. + + bool isExtract = ((extractStatuses[i] & kStatus_Extract) != 0); Int32 askMode = - ((extractStatuses[i] & kStatus_Extract) != 0) ? (testMode ? + isExtract ? (testMode ? NExtract::NAskMode::kTest : NExtract::NAskMode::kExtract) : NExtract::NAskMode::kSkip; unpacker.linkFile = NULL; - if (((extractStatuses[i] & kStatus_Link) != 0)) + // if (!testMode) + if ((extractStatuses[i] & kStatus_Link) != 0) { int bufIndex = FindLinkBuf(linkFiles, i); if (bufIndex < 0) @@ -2791,13 +2789,12 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, RINOK(extractCallback->GetStream(index, &realOutStream, askMode)); - bool isSolid; + bool isSolid = false; + if (!item->IsService()) { - bool &needClearSolid = unpacker.NeedClearSolid[item->IsService() ? 1 : 0]; - isSolid = (item->IsSolid() && !needClearSolid); - if (item->IsService()) - isSolid = false; - needClearSolid = !item->IsSolid(); + if (item->IsSolid()) + isSolid = unpacker.SolidAllowed; + unpacker.SolidAllowed = isSolid; } if (item->IsDir()) @@ -2826,18 +2823,30 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, curUnpackSize = 0; curPackSize = GetPackSize(index2); } - else if ((unsigned)index2 < index) - bufIndex = FindLinkBuf(linkFiles, index2); + else + { + if ((unsigned)index2 < index) + bufIndex = FindLinkBuf(linkFiles, index2); + } } + bool needCallback = true; + if (!realOutStream) { if (testMode) { if (item->NeedUse_as_CopyLink_or_HardLink()) { + Int32 opRes = NExtract::NOperationResult::kOK; + if (bufIndex >= 0) + { + const CLinkFile &linkFile = linkFiles[bufIndex]; + opRes = DecoderRes_to_OpRes(linkFile.Res, linkFile.crcOK); + } + RINOK(extractCallback->PrepareOperation(askMode)); - RINOK(extractCallback->SetOperationResult(NExtract::NOperationResult::kOK)); + RINOK(extractCallback->SetOperationResult(opRes)); continue; } } @@ -2846,10 +2855,10 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, if (item->IsService()) continue; - if (item->NeedUse_as_HardLink()) - continue; + needCallback = false; - bool needDecode = false; + if (!item->NeedUse_as_HardLink()) + if (index2 < 0) for (unsigned n = i + 1; n < _refs.Size(); n++) { @@ -2860,41 +2869,56 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, break; if (extractStatuses[i] != 0) { - needDecode = true; + needCallback = true; break; } } - if (!needDecode) - continue; - askMode = NExtract::NAskMode::kSkip; } } - RINOK(extractCallback->PrepareOperation(askMode)); + if (needCallback) + { + RINOK(extractCallback->PrepareOperation(askMode)); + } if (bufIndex >= 0) { CLinkFile &linkFile = linkFiles[bufIndex]; - if (linkFile.NumLinks == 0) - return E_FAIL; - if (realOutStream) + + if (isExtract) { - RINOK(CopyData_with_Progress(linkFile.Data, linkFile.Data.Size(), realOutStream, progress)); + if (linkFile.NumLinks == 0) + return E_FAIL; + + if (needCallback) + if (realOutStream) + { + RINOK(CopyData_with_Progress(linkFile.Data, linkFile.Data.Size(), realOutStream, progress)); + } + + if (--linkFile.NumLinks == 0) + linkFile.Data.Free(); + } + + if (needCallback) + { + RINOK(extractCallback->SetOperationResult(DecoderRes_to_OpRes(linkFile.Res, linkFile.crcOK))); } - if (--linkFile.NumLinks == 0) - linkFile.Data.Free(); - RINOK(extractCallback->SetOperationResult(DecoderRes_to_OpRes(linkFile.Res, linkFile.crcOK))); continue; } + if (!needCallback) + continue; + if (item->NeedUse_as_CopyLink()) { - RINOK(extractCallback->SetOperationResult( - realOutStream ? - NExtract::NOperationResult::kUnsupportedMethod: - NExtract::NOperationResult::kOK)); + int opRes = realOutStream ? + NExtract::NOperationResult::kUnsupportedMethod: + NExtract::NOperationResult::kOK; + realOutStream.Release(); + RINOK(extractCallback->SetOperationResult(opRes)); continue; } @@ -2941,7 +2965,7 @@ STDMETHODIMP CHandler::Extract(const UInt32 *indices, UInt32 numItems, } { - FOR_VECTOR(i, linkFiles) + FOR_VECTOR (i, linkFiles) if (linkFiles[i].NumLinks != 0) return E_FAIL; } diff --git a/CPP/7zip/Archive/Rar/RarHandler.cpp b/CPP/7zip/Archive/Rar/RarHandler.cpp index e23c4a96..70ea58fa 100644 --- a/CPP/7zip/Archive/Rar/RarHandler.cpp +++ b/CPP/7zip/Archive/Rar/RarHandler.cpp @@ -6,6 +6,7 @@ #include "../../../Common/ComTry.h" #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/UTFConvert.h" #include "../../../Windows/PropVariantUtils.h" @@ -136,8 +137,7 @@ class CInArchive NHeader::NBlock::CBlock m_BlockHeader; NCrypto::NRar3::CDecoder *m_RarAESSpec; CMyComPtr m_RarAES; - CByteBuffer m_DecryptedData; - Byte *m_DecryptedDataAligned; + CAlignedBuffer m_DecryptedDataAligned; UInt32 m_DecryptedDataSize; bool m_CryptoMode; UInt32 m_CryptoPos; @@ -415,6 +415,8 @@ bool CInArchive::ReadHeaderReal(const Byte *p, unsigned size, CItem &item) if (size < 8) return false; item.PackSize |= ((UInt64)Get32(p) << 32); + if (item.PackSize >= ((UInt64)1 << 63)) + return false; item.Size |= ((UInt64)Get32(p + 4) << 32); p += 8; size -= 8; @@ -551,11 +553,12 @@ HRESULT CInArchive::GetNextItem(CItem &item, ICryptoGetTextPassword *getTextPass m_RarAESSpec->SetPassword((const Byte *)buffer, len * 2); const UInt32 kDecryptedBufferSize = (1 << 12); - if (m_DecryptedData.Size() == 0) + if (m_DecryptedDataAligned.Size() == 0) { - const UInt32 kAlign = 16; - m_DecryptedData.Alloc(kDecryptedBufferSize + kAlign); - m_DecryptedDataAligned = (Byte *)((ptrdiff_t)((Byte *)m_DecryptedData + kAlign - 1) & ~(ptrdiff_t)(kAlign - 1)); + // const UInt32 kAlign = 16; + m_DecryptedDataAligned.AllocAtLeast(kDecryptedBufferSize); + if (!m_DecryptedDataAligned.IsAllocated()) + return E_OUTOFMEMORY; } RINOK(m_RarAES->Init()); size_t decryptedDataSizeT = kDecryptedBufferSize; @@ -667,7 +670,8 @@ HRESULT CInArchive::GetNextItem(CItem &item, ICryptoGetTextPassword *getTextPass { if (processed < offset + 2) error = k_ErrorType_Corrupted; - ArcInfo.VolNumber = (UInt32)Get16(m_FileHeaderData + offset); + else + ArcInfo.VolNumber = (UInt32)Get16(m_FileHeaderData + offset); } ArcInfo.EndOfArchive_was_Read = true; diff --git a/CPP/7zip/Archive/SquashfsHandler.cpp b/CPP/7zip/Archive/SquashfsHandler.cpp index b1d7a24c..f0d4fdb9 100644 --- a/CPP/7zip/Archive/SquashfsHandler.cpp +++ b/CPP/7zip/Archive/SquashfsHandler.cpp @@ -4,6 +4,7 @@ #include "../../../C/Alloc.h" #include "../../../C/CpuArch.h" +#include "../../../C/LzmaDec.h" #include "../../../C/Xz.h" #include "../../Common/ComTry.h" @@ -24,7 +25,7 @@ #include "../Compress/CopyCoder.h" #include "../Compress/ZlibDecoder.h" -#include "../Compress/LzmaDecoder.h" +// #include "../Compress/LzmaDecoder.h" namespace NArchive { namespace NSquashfs { @@ -866,8 +867,8 @@ class CHandler: CBufPtrSeqOutStream *_outStreamSpec; CMyComPtr _outStream; - NCompress::NLzma::CDecoder *_lzmaDecoderSpec; - CMyComPtr _lzmaDecoder; + // NCompress::NLzma::CDecoder *_lzmaDecoderSpec; + // CMyComPtr _lzmaDecoder; NCompress::NZlib::CDecoder *_zlibDecoderSpec; CMyComPtr _zlibDecoder; @@ -1155,12 +1156,13 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (inSize != _zlibDecoderSpec->GetInputProcessedSize()) return S_FALSE; } + /* else if (method == kMethod_LZMA) { if (!_lzmaDecoder) { _lzmaDecoderSpec = new NCompress::NLzma::CDecoder(); - _lzmaDecoderSpec->FinishStream = true; + // _lzmaDecoderSpec->FinishStream = true; _lzmaDecoder = _lzmaDecoderSpec; } const UInt32 kPropsSize = LZMA_PROPS_SIZE + 8; @@ -1187,6 +1189,7 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (inSize != propsSize + _lzmaDecoderSpec->GetInputProcessedSize()) return S_FALSE; } + */ else { if (_inputBuffer.Size() < inSize) @@ -1200,11 +1203,49 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (!dest) return E_OUTOFMEMORY; } + SizeT destLen = outSizeMax, srcLen = inSize; + if (method == kMethod_LZO) { RINOK(LzoDecode(dest, &destLen, _inputBuffer, &srcLen)); } + else if (method == kMethod_LZMA) + { + Byte props[5]; + const Byte *src = _inputBuffer; + + if (_noPropsLZMA) + { + props[0] = 0x5D; + SetUi32(&props[1], _h.BlockSize); + } + else + { + const UInt32 kPropsSize = LZMA_PROPS_SIZE + 8; + if (inSize < kPropsSize) + return S_FALSE; + memcpy(props, src, LZMA_PROPS_SIZE); + UInt64 outSize = GetUi64(src + LZMA_PROPS_SIZE); + if (outSize > outSizeMax) + return S_FALSE; + destLen = (SizeT)outSize; + src += kPropsSize; + inSize -= kPropsSize; + srcLen = inSize; + } + + ELzmaStatus status; + SRes res = LzmaDecode(dest, &destLen, + src, &srcLen, + props, LZMA_PROPS_SIZE, + LZMA_FINISH_END, + &status, &g_Alloc); + if (res != 0) + return SResToHRESULT(res); + if (status != LZMA_STATUS_FINISHED_WITH_MARK) + return S_FALSE; + } else { ECoderStatus status; @@ -1217,6 +1258,7 @@ HRESULT CHandler::Decompress(ISequentialOutStream *outStream, Byte *outBuf, bool if (status != CODER_STATUS_NEEDS_MORE_INPUT || !XzUnpacker_IsStreamWasFinished(&_xz)) return S_FALSE; } + if (inSize != srcLen) return S_FALSE; if (outBuf) diff --git a/CPP/7zip/Archive/SwfHandler.cpp b/CPP/7zip/Archive/SwfHandler.cpp index 1720e7bf..224ab152 100644 --- a/CPP/7zip/Archive/SwfHandler.cpp +++ b/CPP/7zip/Archive/SwfHandler.cpp @@ -46,7 +46,7 @@ static const Byte SWF_COMPRESSED_LZMA = 'Z'; static const Byte SWF_MIN_COMPRESSED_ZLIB_VER = 6; static const Byte SWF_MIN_COMPRESSED_LZMA_VER = 13; -static const Byte kVerLim = 20; +static const Byte kVerLim = 64; API_FUNC_static_IsArc IsArc_Swf(const Byte *p, size_t size) { diff --git a/CPP/7zip/Archive/Tar/TarIn.cpp b/CPP/7zip/Archive/Tar/TarIn.cpp index a3318685..3ef2ff8c 100644 --- a/CPP/7zip/Archive/Tar/TarIn.cpp +++ b/CPP/7zip/Archive/Tar/TarIn.cpp @@ -443,14 +443,14 @@ HRESULT ReadItem(ISequentialInStream *stream, bool &filled, CItemEx &item, EErro case 'X': { // pax Extended Header - if (item.Name.IsPrefixedBy("PaxHeader/")) + if (item.Name.IsPrefixedBy("PaxHeader/") + || item.Name.Find("PaxHeaders.4467/") >= 0) { RINOK(ReadDataToString(stream, item, pax, error)); if (error != k_ErrorType_OK) return S_OK; continue; } - break; } case NFileHeader::NLinkFlag::kDumpDir: @@ -488,7 +488,11 @@ HRESULT ReadItem(ISequentialInStream *stream, bool &filled, CItemEx &item, EErro if (ParsePaxLongName(pax, name)) item.Name = name; else - error = k_ErrorType_Warning; + { + // no "path" property is allowed in pax4467 + // error = k_ErrorType_Warning; + } + pax.Empty(); } return S_OK; diff --git a/CPP/7zip/Archive/UefiHandler.cpp b/CPP/7zip/Archive/UefiHandler.cpp index f619591c..e67c9465 100644 --- a/CPP/7zip/Archive/UefiHandler.cpp +++ b/CPP/7zip/Archive/UefiHandler.cpp @@ -791,20 +791,24 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val void CHandler::AddCommentString(const char *name, UInt32 pos) { UString s; - const Byte *buf = _bufs[0]; if (pos < _h.HeaderSize) return; - for (UInt32 i = pos;; i += 2) + if (pos >= _h.OffsetToCapsuleBody) + return; + UInt32 limit = (_h.OffsetToCapsuleBody - pos) & ~(UInt32)1; + const Byte *buf = _bufs[0] + pos; + for (UInt32 i = 0;;) { - if (s.Len() > (1 << 16) || i >= _h.OffsetToCapsuleBody) + if (s.Len() > (1 << 16) || i >= limit) return; wchar_t c = Get16(buf + i); + i += 2; if (c == 0) { - i += 2; - if (i >= _h.OffsetToCapsuleBody) + if (i >= limit) return; c = Get16(buf + i); + i += 2; if (c == 0) break; s.Add_LF(); diff --git a/CPP/7zip/Archive/Wim/WimHandlerOut.cpp b/CPP/7zip/Archive/Wim/WimHandlerOut.cpp index 0d29ebf0..874b6c7f 100644 --- a/CPP/7zip/Archive/Wim/WimHandlerOut.cpp +++ b/CPP/7zip/Archive/Wim/WimHandlerOut.cpp @@ -671,7 +671,7 @@ void CHeader::SetDefaultFields(bool useLZX) ChunkSize = kChunkSize; ChunkSizeBits = kChunkSizeBits; } - g_RandomGenerator.Generate(Guid, 16); + MY_RAND_GEN(Guid, 16); PartNumber = 1; NumParts = 1; NumImages = 1; diff --git a/CPP/7zip/Archive/Wim/WimIn.cpp b/CPP/7zip/Archive/Wim/WimIn.cpp index 90503ab6..17948159 100644 --- a/CPP/7zip/Archive/Wim/WimIn.cpp +++ b/CPP/7zip/Archive/Wim/WimIn.cpp @@ -111,7 +111,9 @@ HRESULT CUnpacker::UnpackChunk( } else if (method == NMethod::kLZX) { - lzxDecoderSpec->SetExternalWindow(unpackBuf.Data, chunkSizeBits); + res = lzxDecoderSpec->SetExternalWindow(unpackBuf.Data, chunkSizeBits); + if (res != S_OK) + return E_NOTIMPL; lzxDecoderSpec->KeepHistoryForNext = false; lzxDecoderSpec->SetKeepHistory(false); res = lzxDecoderSpec->Code(packBuf.Data, inSize, (UInt32)outSize); @@ -563,7 +565,13 @@ void CDatabase::GetItemPath(unsigned index1, bool showImageNumber, NWindows::NCO wchar_t *dest = s + size; meta += 2; for (unsigned i = 0; i < len; i++) - dest[i] = Get16(meta + i * 2); + { + wchar_t c = Get16(meta + i * 2); + // 18.06 + if (c == CHAR_PATH_SEPARATOR || c == '/') + c = '_'; + dest[i] = c; + } } if (index < 0) return; @@ -866,7 +874,11 @@ HRESULT CDatabase::ParseImageDirs(CByteBuffer &buf, int parent) if (DirProcessed == DirSize - 8 && Get64(p + DirSize - 8) != 0) return S_OK; - return S_FALSE; + // 18.06: we support cases, when some old dism can capture images + // where DirProcessed much smaller than DirSize + HeadersError = true; + return S_OK; + // return S_FALSE; } diff --git a/CPP/7zip/Archive/Wim/WimIn.h b/CPP/7zip/Archive/Wim/WimIn.h index 54e1ed53..e4157df5 100644 --- a/CPP/7zip/Archive/Wim/WimIn.h +++ b/CPP/7zip/Archive/Wim/WimIn.h @@ -468,7 +468,7 @@ public: int ExludedItem; // -1 : if there are no exclude items CUIntVector VirtualRoots; // we use them for old 1.10 WIM archives - bool ThereIsError() const { return RefCountError; } + bool ThereIsError() const { return RefCountError || HeadersError; } unsigned GetNumUserItemsInImage(unsigned imageIndex) const { @@ -544,7 +544,10 @@ public: HeadersError = false; } - CDatabase(): RefCountError(false) {} + CDatabase(): + RefCountError(false), + HeadersError(false) + {} void GetShortName(unsigned index, NWindows::NCOM::CPropVariant &res) const; void GetItemName(unsigned index1, NWindows::NCOM::CPropVariant &res) const; diff --git a/CPP/7zip/Archive/XzHandler.cpp b/CPP/7zip/Archive/XzHandler.cpp index 74cbca9d..743271a3 100644 --- a/CPP/7zip/Archive/XzHandler.cpp +++ b/CPP/7zip/Archive/XzHandler.cpp @@ -488,7 +488,7 @@ HRESULT CHandler::Open2(IInStream *inStream, /* UInt32 flags, */ IArchiveOpenCal { CXzBlock block; - Bool isIndex; + BoolInt isIndex; UInt32 headerSizeRes; SRes res2 = XzBlock_ReadHeader(&block, &inStreamWrap.vt, &isIndex, &headerSizeRes); if (res2 == SZ_OK && !isIndex) @@ -820,7 +820,7 @@ static HRESULT DecodeBlock(CXzUnpackerCPP2 &xzu, packRem -= inLen; - Bool blockFinished = XzUnpacker_IsBlockFinished(&xzu.p); + BoolInt blockFinished = XzUnpacker_IsBlockFinished(&xzu.p); if ((inLen == 0 && outLen == 0) || blockFinished) { diff --git a/CPP/7zip/Archive/Zip/ZipHandler.cpp b/CPP/7zip/Archive/Zip/ZipHandler.cpp index 528313a7..cea766d7 100644 --- a/CPP/7zip/Archive/Zip/ZipHandler.cpp +++ b/CPP/7zip/Archive/Zip/ZipHandler.cpp @@ -599,8 +599,12 @@ STDMETHODIMP CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *val case kpidHostOS: { - const Byte hostOS = item.GetHostOS(); - TYPE_TO_PROP(kHostOS, hostOS, prop); + if (item.FromCentral) + { + // 18.06: now we use HostOS only from Central::MadeByVersion + const Byte hostOS = item.MadeByVersion.HostOS; + TYPE_TO_PROP(kHostOS, hostOS, prop); + } break; } diff --git a/CPP/7zip/Archive/Zip/ZipIn.cpp b/CPP/7zip/Archive/Zip/ZipIn.cpp index dc77757e..047d3d2e 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.cpp +++ b/CPP/7zip/Archive/Zip/ZipIn.cpp @@ -155,6 +155,7 @@ void CInArchive::Close() HeadersError = false; HeadersWarning = false; ExtraMinorError = false; + UnexpectedEnd = false; LocalsWereRead = false; LocalsCenterMerged = false; @@ -1729,6 +1730,9 @@ HRESULT CInArchive::FindCd(bool checkOffsetMode) HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdInfo, UInt64 cdOffset, UInt64 cdSize) { items.Clear(); + + // _startLocalFromCd_Disk = (UInt32)(Int32)-1; + // _startLocalFromCd_Offset = (UInt64)(Int64)-1; RINOK(SeekToVol(IsMultiVol ? cdInfo.CdDisk : -1, cdOffset)); @@ -1752,6 +1756,17 @@ HRESULT CInArchive::TryReadCd(CObjectVector &items, const CCdInfo &cdIn { CItemEx cdItem; RINOK(ReadCdItem(cdItem)); + + /* + if (cdItem.Disk < _startLocalFromCd_Disk || + cdItem.Disk == _startLocalFromCd_Disk && + cdItem.LocalHeaderPos < _startLocalFromCd_Offset) + { + _startLocalFromCd_Disk = cdItem.Disk; + _startLocalFromCd_Offset = cdItem.LocalHeaderPos; + } + */ + items.Add(cdItem); } if (Callback && (items.Size() & 0xFFF) == 0) @@ -2509,6 +2524,8 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) { ArcInfo.CdWasRead = true; ArcInfo.FirstItemRelatOffset = items[0].LocalHeaderPos; + + // ArcInfo.FirstItemRelatOffset = _startLocalFromCd_Offset; } } } @@ -2535,6 +2552,10 @@ HRESULT CInArchive::ReadHeaders(CObjectVector &items) items.Clear(); localsWereRead = true; + HeadersError = false; + HeadersWarning = false; + ExtraMinorError = false; + // we can use any mode: with buffer and without buffer // without buffer : skips packed data : fast for big files : slow for small files // with buffer : reads packed data : slow for big files : fast for small files diff --git a/CPP/7zip/Archive/Zip/ZipIn.h b/CPP/7zip/Archive/Zip/ZipIn.h index f43f0396..61f680d8 100644 --- a/CPP/7zip/Archive/Zip/ZipIn.h +++ b/CPP/7zip/Archive/Zip/ZipIn.h @@ -250,6 +250,9 @@ class CInArchive UInt64 _streamPos; UInt64 _cnt; + // UInt32 _startLocalFromCd_Disk; + // UInt64 _startLocalFromCd_Offset; + size_t GetAvail() const { return _bufCached - _bufPos; } void InitBuf() { _bufPos = 0; _bufCached = 0; } @@ -383,6 +386,9 @@ public: UInt64 GetEmbeddedStubSize() const { + // it's possible that first item in CD doesn refers to first local item + // so FirstItemRelatOffset is not first local item + if (ArcInfo.CdWasRead) return ArcInfo.FirstItemRelatOffset; if (IsMultiVol) diff --git a/CPP/7zip/Archive/Zip/ZipItem.h b/CPP/7zip/Archive/Zip/ZipItem.h index 38b44c12..e9c6392a 100644 --- a/CPP/7zip/Archive/Zip/ZipItem.h +++ b/CPP/7zip/Archive/Zip/ZipItem.h @@ -214,6 +214,12 @@ class CLocalItem public: UInt16 Flags; UInt16 Method; + + /* + Zip specification doesn't mention that ExtractVersion field uses HostOS subfield. + 18.06: 7-Zip now doesn't use ExtractVersion::HostOS to detect codePage + */ + CVersion ExtractVersion; UInt64 Size; @@ -309,7 +315,8 @@ public: UInt32 GetWinAttrib() const; bool GetPosixAttrib(UInt32 &attrib) const; - Byte GetHostOS() const { return FromCentral ? MadeByVersion.HostOS : ExtractVersion.HostOS; } + // 18.06: 0 instead of ExtractVersion.HostOS for local item + Byte GetHostOS() const { return FromCentral ? MadeByVersion.HostOS : (Byte)0; } void GetUnicodeString(UString &res, const AString &s, bool isComment, bool useSpecifiedCodePage, UINT codePage) const; @@ -326,7 +333,10 @@ public: UINT GetCodePage() const { - Byte hostOS = GetHostOS(); + // 18.06: now we use HostOS only from Central::MadeByVersion + if (!FromCentral) + return CP_OEMCP; + Byte hostOS = MadeByVersion.HostOS; return (UINT)(( hostOS == NFileHeader::NHostOS::kFAT || hostOS == NFileHeader::NHostOS::kNTFS diff --git a/CPP/7zip/Archive/Zip/ZipUpdate.cpp b/CPP/7zip/Archive/Zip/ZipUpdate.cpp index 3ceea97b..5b7d5928 100644 --- a/CPP/7zip/Archive/Zip/ZipUpdate.cpp +++ b/CPP/7zip/Archive/Zip/ZipUpdate.cpp @@ -40,7 +40,10 @@ static const Byte kHostOS = #endif static const Byte kMadeByHostOS = kHostOS; -static const Byte kExtractHostOS = kHostOS; + +// 18.06: now we always write zero to high byte of ExtractVersion field. +// Previous versions of p7zip wrote (NFileHeader::NHostOS::kUnix) there, that is not correct +static const Byte kExtractHostOS = 0; static const Byte kMethodForDirectory = NFileHeader::NCompressionMethod::kStore; diff --git a/CPP/7zip/Bundles/Alone/Alone.dsp b/CPP/7zip/Bundles/Alone/Alone.dsp index 9b5d7531..6b0df241 100644 --- a/CPP/7zip/Bundles/Alone/Alone.dsp +++ b/CPP/7zip/Bundles/Alone/Alone.dsp @@ -2500,6 +2500,15 @@ SOURCE=..\..\..\..\C\Delta.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\HuffEnc.c !IF "$(CFG)" == "Alone - Win32 Release" diff --git a/CPP/7zip/Bundles/Alone/makefile b/CPP/7zip/Bundles/Alone/makefile index f68a4296..ac1445e9 100644 --- a/CPP/7zip/Bundles/Alone/makefile +++ b/CPP/7zip/Bundles/Alone/makefile @@ -3,10 +3,9 @@ MY_CONSOLE = 1 CFLAGS = $(CFLAGS) !IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -DZSTD_MULTITHREAD +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE !ENDIF - COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ @@ -291,6 +290,8 @@ ZSTD_OBJS = \ $O\xxhash.obj \ $O\zstd_common.obj \ $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ diff --git a/CPP/7zip/Bundles/Alone7z/Alone.dsp b/CPP/7zip/Bundles/Alone7z/Alone.dsp index 6147e105..e12207db 100644 --- a/CPP/7zip/Bundles/Alone7z/Alone.dsp +++ b/CPP/7zip/Bundles/Alone7z/Alone.dsp @@ -1646,6 +1646,15 @@ SOURCE=..\..\..\..\C\Delta.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\IStream.h # End Source File # Begin Source File diff --git a/CPP/7zip/Bundles/Alone7z/makefile b/CPP/7zip/Bundles/Alone7z/makefile index 91ee1922..4f23c10c 100644 --- a/CPP/7zip/Bundles/Alone7z/makefile +++ b/CPP/7zip/Bundles/Alone7z/makefile @@ -1,10 +1,6 @@ PROG = 7zr.exe -MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) -DPROG_VARIANT_R - -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -!ENDIF +CFLAGS = $(CFLAGS) \ + -DPROG_VARIANT_R \ COMMON_OBJS = \ $O\CommandLineParser.obj \ diff --git a/CPP/7zip/Bundles/Codec_brotli/makefile b/CPP/7zip/Bundles/Codec_brotli/makefile index 9a769011..f319fe15 100644 --- a/CPP/7zip/Bundles/Codec_brotli/makefile +++ b/CPP/7zip/Bundles/Codec_brotli/makefile @@ -1,6 +1,6 @@ PROG = brotli.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID +CFLAGS = $(CFLAGS) 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ diff --git a/CPP/7zip/Bundles/Codec_brotli/resource.rc b/CPP/7zip/Bundles/Codec_brotli/resource.rc index fba52315..a29be04e 100644 --- a/CPP/7zip/Bundles/Codec_brotli/resource.rc +++ b/CPP/7zip/Bundles/Codec_brotli/resource.rc @@ -1,6 +1,6 @@ #include "../../../../C/7zVersionTr.h" #include "../../../../C/7zVersion.rc" -MY_VERSION_INFO_DLL("7-Zip Brotli Plugin v1.0.6", "Brotli") +MY_VERSION_INFO_DLL("7-Zip Brotli Plugin v1.0.7", "Brotli") 101 ICON "../../Archive/Icons/7z.ico" diff --git a/CPP/7zip/Bundles/Codec_flzma2/makefile b/CPP/7zip/Bundles/Codec_flzma2/makefile index ef34c5d4..3a544e6c 100644 --- a/CPP/7zip/Bundles/Codec_flzma2/makefile +++ b/CPP/7zip/Bundles/Codec_flzma2/makefile @@ -1,6 +1,6 @@ PROG = flzma2.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DNO_XXHASH +CFLAGS = $(CFLAGS) -DNO_XXHASH 7ZIP_COMMON_OBJS = \ $O\CWrappers.obj \ diff --git a/CPP/7zip/Bundles/Codec_lizard/makefile b/CPP/7zip/Bundles/Codec_lizard/makefile index a97e5e41..093fb3ae 100644 --- a/CPP/7zip/Bundles/Codec_lizard/makefile +++ b/CPP/7zip/Bundles/Codec_lizard/makefile @@ -1,6 +1,6 @@ PROG = lizard.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID +CFLAGS = $(CFLAGS) 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ diff --git a/CPP/7zip/Bundles/Codec_lz4/makefile b/CPP/7zip/Bundles/Codec_lz4/makefile index cf20a8df..1f339bec 100644 --- a/CPP/7zip/Bundles/Codec_lz4/makefile +++ b/CPP/7zip/Bundles/Codec_lz4/makefile @@ -1,6 +1,6 @@ PROG = lz4.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID +CFLAGS = $(CFLAGS) 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ diff --git a/CPP/7zip/Bundles/Codec_lz5/makefile b/CPP/7zip/Bundles/Codec_lz5/makefile index 8e620a88..68450a85 100644 --- a/CPP/7zip/Bundles/Codec_lz5/makefile +++ b/CPP/7zip/Bundles/Codec_lz5/makefile @@ -1,6 +1,6 @@ PROG = lz5.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID +CFLAGS = $(CFLAGS) 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ diff --git a/CPP/7zip/Bundles/Codec_zstd/makefile b/CPP/7zip/Bundles/Codec_zstd/makefile index f6fb820f..62c0f2fe 100644 --- a/CPP/7zip/Bundles/Codec_zstd/makefile +++ b/CPP/7zip/Bundles/Codec_zstd/makefile @@ -1,6 +1,6 @@ -PROG = zstd-min.dll +PROG = zstd.dll DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_MULTITHREAD +CFLAGS = $(CFLAGS) -DZSTD_LEGACY_SUPPORT -DZSTD_MULTITHREAD 7ZIP_COMMON_OBJS = \ $O\StreamUtils.obj \ @@ -35,6 +35,8 @@ ZSTD_OBJS = \ $O\xxhash.obj \ $O\zstd_common.obj \ $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ @@ -42,6 +44,13 @@ ZSTD_OBJS = \ $O\zstd_ldm.obj \ $O\zstdmt_compress.obj \ $O\zstd_opt.obj \ + $O\zstd_v01.obj \ + $O\zstd_v02.obj \ + $O\zstd_v03.obj \ + $O\zstd_v04.obj \ + $O\zstd_v05.obj \ + $O\zstd_v06.obj \ + $O\zstd_v07.obj \ ZSTDMT_OBJS = \ $O\zstd-mt_threading.obj \ diff --git a/CPP/7zip/Bundles/Codec_zstd/resource.rc b/CPP/7zip/Bundles/Codec_zstd/resource.rc index 5d17e2cc..a53534ce 100644 --- a/CPP/7zip/Bundles/Codec_zstd/resource.rc +++ b/CPP/7zip/Bundles/Codec_zstd/resource.rc @@ -1,6 +1,6 @@ #include "../../../../C/7zVersionTr.h" #include "../../../../C/7zVersion.rc" -MY_VERSION_INFO_DLL("7-Zip ZS Plugin v1.3.7", "zstd") +MY_VERSION_INFO_DLL("7-Zip ZS Plugin v1.3.8", "zstd") 101 ICON "../../Archive/Icons/zst.ico" diff --git a/CPP/7zip/Bundles/Codec_zstdF/StdAfx.cpp b/CPP/7zip/Bundles/Codec_zstdF/StdAfx.cpp deleted file mode 100644 index c6d3b1fa..00000000 --- a/CPP/7zip/Bundles/Codec_zstdF/StdAfx.cpp +++ /dev/null @@ -1,3 +0,0 @@ -// StdAfx.cpp - -#include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Codec_zstdF/StdAfx.h b/CPP/7zip/Bundles/Codec_zstdF/StdAfx.h deleted file mode 100644 index 59d9ac15..00000000 --- a/CPP/7zip/Bundles/Codec_zstdF/StdAfx.h +++ /dev/null @@ -1,8 +0,0 @@ -// StdAfx.h - -#ifndef __STDAFX_H -#define __STDAFX_H - -#include "../../../Common/Common.h" - -#endif diff --git a/CPP/7zip/Bundles/Codec_zstdF/makefile b/CPP/7zip/Bundles/Codec_zstdF/makefile deleted file mode 100644 index d8857707..00000000 --- a/CPP/7zip/Bundles/Codec_zstdF/makefile +++ /dev/null @@ -1,56 +0,0 @@ -PROG = zstd.dll -DEF_FILE = ../../Compress/Codec.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_LEGACY_SUPPORT -DZSTD_MULTITHREAD - -7ZIP_COMMON_OBJS = \ - $O\StreamUtils.obj \ - -WIN_OBJS = \ - $O\System.obj \ - -COMPRESS_OBJS = \ - $O\CodecExports.obj \ - $O\DllExportsCompress.obj \ - -C_OBJS = \ - $O\Alloc.obj \ - $O\Threads.obj \ - -COMPRESS_OBJS = $(COMPRESS_OBJS) \ - $O\ZstdDecoder.obj \ - $O\ZstdEncoder.obj \ - $O\ZstdRegister.obj \ - -ZSTD_OBJS = \ - $O\debug.obj \ - $O\entropy_common.obj \ - $O\error_private.obj \ - $O\fse_compress.obj \ - $O\fse_decompress.obj \ - $O\hist.obj \ - $O\huf_compress.obj \ - $O\huf_decompress.obj \ - $O\pool.obj \ - $O\threading.obj \ - $O\xxhash.obj \ - $O\zstd_common.obj \ - $O\zstd_compress.obj \ - $O\zstd_decompress.obj \ - $O\zstd_double_fast.obj \ - $O\zstd_fast.obj \ - $O\zstd_lazy.obj \ - $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ - $O\zstd_opt.obj \ - $O\zstd_v01.obj \ - $O\zstd_v02.obj \ - $O\zstd_v03.obj \ - $O\zstd_v04.obj \ - $O\zstd_v05.obj \ - $O\zstd_v06.obj \ - $O\zstd_v07.obj \ - -ZSTDMT_OBJS = \ - $O\zstd-mt_threading.obj \ - -!include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Codec_zstdF/resource.rc b/CPP/7zip/Bundles/Codec_zstdF/resource.rc deleted file mode 100644 index 5d17e2cc..00000000 --- a/CPP/7zip/Bundles/Codec_zstdF/resource.rc +++ /dev/null @@ -1,6 +0,0 @@ -#include "../../../../C/7zVersionTr.h" -#include "../../../../C/7zVersion.rc" - -MY_VERSION_INFO_DLL("7-Zip ZS Plugin v1.3.7", "zstd") - -101 ICON "../../Archive/Icons/zst.ico" diff --git a/CPP/7zip/Bundles/Fm/FM.dsp b/CPP/7zip/Bundles/Fm/FM.dsp index 08994110..ff6208f9 100644 --- a/CPP/7zip/Bundles/Fm/FM.dsp +++ b/CPP/7zip/Bundles/Fm/FM.dsp @@ -985,6 +985,15 @@ SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\LzFind.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/Bundles/Fm/makefile b/CPP/7zip/Bundles/Fm/makefile index 01ef54dd..b89d2f8f 100644 --- a/CPP/7zip/Bundles/Fm/makefile +++ b/CPP/7zip/Bundles/Fm/makefile @@ -1,14 +1,4 @@ PROG = 7zFM.exe -CFLAGS = $(CFLAGS) \ - -DLANG \ - -DNEW_FOLDER_INTERFACE \ - -!IFDEF UNDER_CE -LIBS = $(LIBS) ceshell.lib Commctrl.lib -!ELSE -LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE -!ENDIF !include "../Format7zF/Arc.mak" diff --git a/CPP/7zip/Bundles/Format7z/makefile b/CPP/7zip/Bundles/Format7z/makefile index 1db7800b..011faaa2 100644 --- a/CPP/7zip/Bundles/Format7z/makefile +++ b/CPP/7zip/Bundles/Format7z/makefile @@ -213,6 +213,8 @@ ZSTD_OBJS = \ $O\xxhash.obj \ $O\zstd_common.obj \ $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ diff --git a/CPP/7zip/Bundles/Format7zExtract/makefile b/CPP/7zip/Bundles/Format7zExtract/makefile index 366c7b39..282bb43c 100644 --- a/CPP/7zip/Bundles/Format7zExtract/makefile +++ b/CPP/7zip/Bundles/Format7zExtract/makefile @@ -1,8 +1,6 @@ PROG = 7zxa.dll DEF_FILE = ../../Archive/Archive2.def -CFLAGS = $(CFLAGS) \ - -DNEED_7ZIP_GUID \ - -DEXTRACT_ONLY \ +CFLAGS = $(CFLAGS) -DEXTRACT_ONLY \ COMMON_OBJS = \ $O\CRC.obj \ @@ -175,6 +173,8 @@ ZSTD_OBJS = \ $O\xxhash.obj \ $O\zstd_common.obj \ $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ diff --git a/CPP/7zip/Bundles/Format7zF/makefile b/CPP/7zip/Bundles/Format7zF/makefile index f23b4250..60799a48 100644 --- a/CPP/7zip/Bundles/Format7zF/makefile +++ b/CPP/7zip/Bundles/Format7zF/makefile @@ -88,6 +88,8 @@ ZSTD_OBJS = \ $O\xxhash.obj \ $O\zstd_common.obj \ $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ $O\zstd_double_fast.obj \ $O\zstd_fast.obj \ diff --git a/CPP/7zip/Bundles/Format7zFO/Arc.mak b/CPP/7zip/Bundles/Format7zFO/Arc.mak deleted file mode 100644 index 9b9ae22a..00000000 --- a/CPP/7zip/Bundles/Format7zFO/Arc.mak +++ /dev/null @@ -1,306 +0,0 @@ -COMMON_OBJS = \ - $O\CRC.obj \ - $O\CrcReg.obj \ - $O\DynLimBuf.obj \ - $O\IntToString.obj \ - $O\Md2Reg.obj \ - $O\Md4Reg.obj \ - $O\Md5Reg.obj \ - $O\MyMap.obj \ - $O\MyString.obj \ - $O\MyVector.obj \ - $O\MyXml.obj \ - $O\NewHandler.obj \ - $O\Sha1Reg.obj \ - $O\Sha256Reg.obj \ - $O\Sha384Reg.obj \ - $O\Sha512Reg.obj \ - $O\StringConvert.obj \ - $O\StringToInt.obj \ - $O\UTFConvert.obj \ - $O\Wildcard.obj \ - $O\XXH32Reg.obj \ - $O\XXH64Reg.obj \ - $O\XzCrc64Init.obj \ - $O\XzCrc64Reg.obj \ - -WIN_OBJS = \ - $O\FileDir.obj \ - $O\FileFind.obj \ - $O\FileIO.obj \ - $O\FileName.obj \ - $O\PropVariant.obj \ - $O\PropVariantUtils.obj \ - $O\Synchronization.obj \ - $O\System.obj \ - $O\TimeUtils.obj \ - -7ZIP_COMMON_OBJS = \ - $O\CreateCoder.obj \ - $O\CWrappers.obj \ - $O\InBuffer.obj \ - $O\InOutTempBuffer.obj \ - $O\FilterCoder.obj \ - $O\LimitedStreams.obj \ - $O\LockedStream.obj \ - $O\MemBlocks.obj \ - $O\MethodId.obj \ - $O\MethodProps.obj \ - $O\OffsetStream.obj \ - $O\OutBuffer.obj \ - $O\OutMemStream.obj \ - $O\ProgressMt.obj \ - $O\ProgressUtils.obj \ - $O\PropId.obj \ - $O\StreamBinder.obj \ - $O\StreamObjects.obj \ - $O\StreamUtils.obj \ - $O\UniqBlocks.obj \ - $O\VirtThread.obj \ - -AR_OBJS = \ - $O\ApmHandler.obj \ - $O\ArHandler.obj \ - $O\ArjHandler.obj \ - $O\Bz2Handler.obj \ - $O\ComHandler.obj \ - $O\CpioHandler.obj \ - $O\CramfsHandler.obj \ - $O\DeflateProps.obj \ - $O\DmgHandler.obj \ - $O\ElfHandler.obj \ - $O\ExtHandler.obj \ - $O\FatHandler.obj \ - $O\FlvHandler.obj \ - $O\GzHandler.obj \ - $O\GptHandler.obj \ - $O\HandlerCont.obj \ - $O\HfsHandler.obj \ - $O\IhexHandler.obj \ - $O\LzHandler.obj \ - $O\Lz4Handler.obj \ - $O\Lz5Handler.obj \ - $O\LizardHandler.obj \ - $O\LzhHandler.obj \ - $O\LzmaHandler.obj \ - $O\MachoHandler.obj \ - $O\MbrHandler.obj \ - $O\MslzHandler.obj \ - $O\MubHandler.obj \ - $O\NtfsHandler.obj \ - $O\PeHandler.obj \ - $O\PpmdHandler.obj \ - $O\QcowHandler.obj \ - $O\RpmHandler.obj \ - $O\SplitHandler.obj \ - $O\SquashfsHandler.obj \ - $O\SwfHandler.obj \ - $O\UefiHandler.obj \ - $O\VdiHandler.obj \ - $O\VhdHandler.obj \ - $O\VmdkHandler.obj \ - $O\XarHandler.obj \ - $O\XzHandler.obj \ - $O\ZHandler.obj \ - $O\ZstdHandler.obj \ - -AR_COMMON_OBJS = \ - $O\CoderMixer2.obj \ - $O\DummyOutStream.obj \ - $O\FindSignature.obj \ - $O\InStreamWithCRC.obj \ - $O\ItemNameUtils.obj \ - $O\MultiStream.obj \ - $O\OutStreamWithCRC.obj \ - $O\OutStreamWithSha1.obj \ - $O\HandlerOut.obj \ - $O\ParseProperties.obj \ - -7Z_OBJS = \ - $O\7zCompressionMode.obj \ - $O\7zDecode.obj \ - $O\7zEncode.obj \ - $O\7zExtract.obj \ - $O\7zFolderInStream.obj \ - $O\7zHandler.obj \ - $O\7zHandlerOut.obj \ - $O\7zHeader.obj \ - $O\7zIn.obj \ - $O\7zOut.obj \ - $O\7zProperties.obj \ - $O\7zSpecStream.obj \ - $O\7zUpdate.obj \ - $O\7zRegister.obj \ - -CAB_OBJS = \ - $O\CabBlockInStream.obj \ - $O\CabHandler.obj \ - $O\CabHeader.obj \ - $O\CabIn.obj \ - $O\CabRegister.obj \ - -CHM_OBJS = \ - $O\ChmHandler.obj \ - $O\ChmIn.obj \ - -ISO_OBJS = \ - $O\IsoHandler.obj \ - $O\IsoHeader.obj \ - $O\IsoIn.obj \ - $O\IsoRegister.obj \ - -NSIS_OBJS = \ - $O\NsisDecode.obj \ - $O\NsisHandler.obj \ - $O\NsisIn.obj \ - $O\NsisRegister.obj \ - -RAR_OBJS = \ - $O\RarHandler.obj \ - $O\Rar5Handler.obj \ - -TAR_OBJS = \ - $O\TarHandler.obj \ - $O\TarHandlerOut.obj \ - $O\TarHeader.obj \ - $O\TarIn.obj \ - $O\TarOut.obj \ - $O\TarUpdate.obj \ - $O\TarRegister.obj \ - -UDF_OBJS = \ - $O\UdfHandler.obj \ - $O\UdfIn.obj \ - -WIM_OBJS = \ - $O\WimHandler.obj \ - $O\WimHandlerOut.obj \ - $O\WimIn.obj \ - $O\WimRegister.obj \ - -ZIP_OBJS = \ - $O\ZipAddCommon.obj \ - $O\ZipHandler.obj \ - $O\ZipHandlerOut.obj \ - $O\ZipIn.obj \ - $O\ZipItem.obj \ - $O\ZipOut.obj \ - $O\ZipUpdate.obj \ - $O\ZipRegister.obj \ - -COMPRESS_OBJS = \ - $O\Bcj2Coder.obj \ - $O\Bcj2Register.obj \ - $O\BcjCoder.obj \ - $O\BcjRegister.obj \ - $O\BitlDecoder.obj \ - $O\BranchMisc.obj \ - $O\BranchRegister.obj \ - $O\ByteSwap.obj \ - $O\BZip2Crc.obj \ - $O\BZip2Decoder.obj \ - $O\BZip2Encoder.obj \ - $O\BZip2Register.obj \ - $O\CopyCoder.obj \ - $O\CopyRegister.obj \ - $O\Deflate64Register.obj \ - $O\DeflateDecoder.obj \ - $O\DeflateEncoder.obj \ - $O\DeflateRegister.obj \ - $O\DeltaFilter.obj \ - $O\ImplodeDecoder.obj \ - $O\LzfseDecoder.obj \ - $O\LzhDecoder.obj \ - $O\Lzma2Decoder.obj \ - $O\Lzma2Encoder.obj \ - $O\Lzma2Register.obj \ - $O\LzmaDecoder.obj \ - $O\LzmaEncoder.obj \ - $O\LzmaRegister.obj \ - $O\LzmsDecoder.obj \ - $O\LzOutWindow.obj \ - $O\LzxDecoder.obj \ - $O\PpmdDecoder.obj \ - $O\PpmdEncoder.obj \ - $O\PpmdRegister.obj \ - $O\PpmdZip.obj \ - $O\QuantumDecoder.obj \ - $O\Rar1Decoder.obj \ - $O\Rar2Decoder.obj \ - $O\Rar3Decoder.obj \ - $O\Rar3Vm.obj \ - $O\Rar5Decoder.obj \ - $O\RarCodecsRegister.obj \ - $O\ShrinkDecoder.obj \ - $O\XpressDecoder.obj \ - $O\XzDecoder.obj \ - $O\XzEncoder.obj \ - $O\ZlibDecoder.obj \ - $O\ZlibEncoder.obj \ - $O\ZDecoder.obj \ - - -CRYPTO_OBJS = \ - $O\7zAes.obj \ - $O\7zAesRegister.obj \ - $O\HmacSha1.obj \ - $O\HmacSha256.obj \ - $O\MyAes.obj \ - $O\MyAesReg.obj \ - $O\Pbkdf2HmacSha1.obj \ - $O\RandGen.obj \ - $O\Rar20Crypto.obj \ - $O\Rar5Aes.obj \ - $O\RarAes.obj \ - $O\WzAes.obj \ - $O\ZipCrypto.obj \ - $O\ZipStrong.obj \ - -HASHES_OBJS = \ - $O\md2.obj \ - $O\md4.obj \ - $O\md5.obj \ - $O\sha512.obj \ - -C_OBJS = \ - $O\7zBuf2.obj \ - $O\7zStream.obj \ - $O\Alloc.obj \ - $O\Bcj2.obj \ - $O\Bcj2Enc.obj \ - $O\Blake2s.obj \ - $O\Bra.obj \ - $O\Bra86.obj \ - $O\BraIA64.obj \ - $O\BwtSort.obj \ - $O\CpuArch.obj \ - $O\Delta.obj \ - $O\HuffEnc.obj \ - $O\LzFind.obj \ - $O\LzFindMt.obj \ - $O\Lzma2Dec.obj \ - $O\Lzma2DecMt.obj \ - $O\Lzma2Enc.obj \ - $O\LzmaDec.obj \ - $O\LzmaEnc.obj \ - $O\MtCoder.obj \ - $O\MtDec.obj \ - $O\Ppmd7.obj \ - $O\Ppmd7Dec.obj \ - $O\Ppmd7Enc.obj \ - $O\Ppmd8.obj \ - $O\Ppmd8Dec.obj \ - $O\Ppmd8Enc.obj \ - $O\Sha1.obj \ - $O\Sha256.obj \ - $O\Sort.obj \ - $O\Threads.obj \ - $O\Xz.obj \ - $O\XzDec.obj \ - $O\XzEnc.obj \ - $O\XzIn.obj \ - -!include "../../Aes.mak" -!include "../../Crc.mak" -!include "../../Crc64.mak" -!include "../../LzmaDec.mak" diff --git a/CPP/7zip/Bundles/Format7zFO/Format7z.dsp b/CPP/7zip/Bundles/Format7zFO/Format7z.dsp deleted file mode 100644 index 39ff535b..00000000 --- a/CPP/7zip/Bundles/Format7zFO/Format7z.dsp +++ /dev/null @@ -1,3008 +0,0 @@ -# Microsoft Developer Studio Project File - Name="7z" - Package Owner=<4> -# Microsoft Developer Studio Generated Build File, Format Version 6.00 -# ** DO NOT EDIT ** - -# TARGTYPE "Win32 (x86) Dynamic-Link Library" 0x0102 - -CFG=7z - Win32 Debug -!MESSAGE This is not a valid makefile. To build this project using NMAKE, -!MESSAGE use the Export Makefile command and run -!MESSAGE -!MESSAGE NMAKE /f "Format7z.mak". -!MESSAGE -!MESSAGE You can specify a configuration when running NMAKE -!MESSAGE by defining the macro CFG on the command line. For example: -!MESSAGE -!MESSAGE NMAKE /f "Format7z.mak" CFG="7z - Win32 Debug" -!MESSAGE -!MESSAGE Possible choices for configuration are: -!MESSAGE -!MESSAGE "7z - Win32 Release" (based on "Win32 (x86) Dynamic-Link Library") -!MESSAGE "7z - Win32 Debug" (based on "Win32 (x86) Dynamic-Link Library") -!MESSAGE - -# Begin Project -# PROP AllowPerConfigDependencies 0 -# PROP Scc_ProjName "" -# PROP Scc_LocalPath "" -CPP=cl.exe -MTL=midl.exe -RSC=rc.exe - -!IF "$(CFG)" == "7z - Win32 Release" - -# PROP BASE Use_MFC 0 -# PROP BASE Use_Debug_Libraries 0 -# PROP BASE Output_Dir "Release" -# PROP BASE Intermediate_Dir "Release" -# PROP BASE Target_Dir "" -# PROP Use_MFC 0 -# PROP Use_Debug_Libraries 0 -# PROP Output_Dir "Release" -# PROP Intermediate_Dir "Release" -# PROP Ignore_Export_Lib 1 -# PROP Target_Dir "" -# ADD BASE CPP /nologo /MT /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "MY7Z_EXPORTS" /YX /FD /c -# ADD CPP /nologo /Gr /MT /W4 /WX /GX /O1 /D "NDEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "MY7Z_EXPORTS" /D "NO_REGISTRY" /D "EXTERNAL_CODECS" /D "_7ZIP_LARGE_PAGES" /D "_7ZIP_ST_9" /FAcs /Yu"StdAfx.h" /FD /GF /c -# ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32 -# ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32 -# ADD BASE RSC /l 0x419 /d "NDEBUG" -# ADD RSC /l 0x419 /d "NDEBUG" -BSC32=bscmake.exe -# ADD BASE BSC32 /nologo -# ADD BSC32 /nologo -LINK32=link.exe -# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"C:\Program Files\7-Zip\7z.dll" /opt:NOWIN98 -# SUBTRACT LINK32 /pdb:none /debug - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# PROP BASE Use_MFC 0 -# PROP BASE Use_Debug_Libraries 1 -# PROP BASE Output_Dir "Debug" -# PROP BASE Intermediate_Dir "Debug" -# PROP BASE Target_Dir "" -# PROP Use_MFC 0 -# PROP Use_Debug_Libraries 1 -# PROP Output_Dir "Debug" -# PROP Intermediate_Dir "Debug" -# PROP Ignore_Export_Lib 1 -# PROP Target_Dir "" -# ADD BASE CPP /nologo /MTd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "MY7Z_EXPORTS" /YX /FD /GZ /c -# ADD CPP /nologo /Gr /MTd /W4 /WX /Gm /GX /ZI /Od /I "..\..\..\..\SDK" /D "_DEBUG" /D "WIN32" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "MY7Z_EXPORTS" /D "NO_REGISTRY" /D "EXTERNAL_CODECS" /D "_7ZIP_LARGE_PAGES" /D "_7ZIP_ST_9" /Yu"StdAfx.h" /FD /GZ /c -# ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32 -# ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32 -# ADD BASE RSC /l 0x419 /d "_DEBUG" -# ADD RSC /l 0x419 /d "_DEBUG" -BSC32=bscmake.exe -# ADD BASE BSC32 /nologo -# ADD BSC32 /nologo -LINK32=link.exe -# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept -# ADD LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"C:\Program Files\7-Zip\7z.dll" /pdbtype:sept /ignore:4033 -# SUBTRACT LINK32 /pdb:none - -!ENDIF - -# Begin Target - -# Name "7z - Win32 Release" -# Name "7z - Win32 Debug" -# Begin Group "Spec" - -# PROP Default_Filter "" -# Begin Group "Icons" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Icons\7z.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\arj.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\bz2.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\cab.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\cpio.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\deb.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\dmg.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\fat.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\gz.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\hfs.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\iso.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\lzh.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\lzma.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\ntfs.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\rar.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\rpm.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\split.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\squashfs.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\tar.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\vhd.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\wim.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\xar.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\xz.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\z.ico -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Icons\zip.ico -# End Source File -# End Group -# Begin Source File - -SOURCE=..\..\Archive\Archive2.def -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ArchiveExports.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\CodecExports.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\DllExports2.cpp -# End Source File -# Begin Source File - -SOURCE=.\resource.rc -# End Source File -# Begin Source File - -SOURCE=.\StdAfx.cpp -# ADD CPP /Yc"StdAfx.h" -# End Source File -# Begin Source File - -SOURCE=.\StdAfx.h -# End Source File -# End Group -# Begin Group "Common" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\..\Common\Common.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\CRC.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\CrcReg.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\Defs.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\DynamicBuffer.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\DynLimBuf.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\DynLimBuf.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\IntToString.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\IntToString.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyBuffer.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyBuffer2.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyCom.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyException.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyInitGuid.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyLinux.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyMap.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyMap.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyString.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyString.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyTypes.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyUnknown.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyVector.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyVector.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyWindows.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyWindows.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyXml.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\MyXml.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\NewHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\NewHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\Sha1Reg.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\Sha256Reg.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\StringConvert.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\StringConvert.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\StringToInt.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\StringToInt.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\UTFConvert.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\UTFConvert.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\Wildcard.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\Wildcard.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\XzCrc64Init.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Common\XzCrc64Reg.cpp -# End Source File -# End Group -# Begin Group "Compress" - -# PROP Default_Filter "" -# Begin Group "Bit Coder" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\BitlDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BitlDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BitlEncoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BitmDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BitmEncoder.h -# End Source File -# End Group -# Begin Group "Rar Compress" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\Rar1Decoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar1Decoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar2Decoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar2Decoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar3Decoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar3Decoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar3Vm.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar3Vm.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar5Decoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Rar5Decoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\RarCodecsRegister.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# End Group -# Begin Group "BZip2 Compress" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\BZip2Const.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Crc.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Crc.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Decoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Decoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Encoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Encoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BZip2Register.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Mtf8.h -# End Source File -# End Group -# Begin Group "Zip Compress" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\Deflate64Register.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeflateConst.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeflateDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeflateDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeflateEncoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeflateEncoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeflateRegister.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ImplodeDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ImplodeDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdZip.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdZip.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ShrinkDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ShrinkDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZlibDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZlibDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZlibEncoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZlibEncoder.h -# End Source File -# End Group -# Begin Group "7z Compress" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\Bcj2Coder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Bcj2Coder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Bcj2Register.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BcjCoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BcjCoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BcjRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BranchMisc.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BranchMisc.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\BranchRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ByteSwap.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\CopyCoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\CopyCoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\CopyRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\DeltaFilter.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Lzma2Decoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Lzma2Decoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Lzma2Encoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Lzma2Encoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\Lzma2Register.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmaDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmaDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmaEncoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmaEncoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmaRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdEncoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdEncoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\PpmdRegister.cpp -# End Source File -# End Group -# Begin Group "Cab Compress" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Compress\Lzx.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzxDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzxDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\QuantumDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\QuantumDecoder.h -# End Source File -# End Group -# Begin Source File - -SOURCE=..\..\Compress\HuffmanDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzfseDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzfseDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzhDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzhDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmsDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzmsDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzOutWindow.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\LzOutWindow.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\XpressDecoder.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\XpressDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\XzDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\XzDecoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\XzEncoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\XzEncoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZDecoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Compress\ZDecoder.h -# End Source File -# End Group -# Begin Group "Crypto" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Crypto\7zAes.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\7zAes.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\7zAesRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\HmacSha1.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\HmacSha1.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\HmacSha256.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\HmacSha256.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\MyAes.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\MyAes.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\MyAesReg.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Pbkdf2HmacSha1.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Pbkdf2HmacSha1.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\RandGen.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\RandGen.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Rar20Crypto.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Rar20Crypto.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Rar5Aes.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Rar5Aes.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\RarAes.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\RarAes.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\Sha1Cls.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\WzAes.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\WzAes.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\ZipCrypto.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\ZipCrypto.h -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\ZipStrong.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Crypto\ZipStrong.h -# End Source File -# End Group -# Begin Group "7zip Common" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Common\CreateCoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\CreateCoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\CWrappers.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\CWrappers.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\FilterCoder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\FilterCoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\InBuffer.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\InBuffer.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\InOutTempBuffer.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\InOutTempBuffer.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\LimitedStreams.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\LimitedStreams.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\MemBlocks.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\MemBlocks.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\MethodId.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\MethodId.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\MethodProps.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\MethodProps.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\OffsetStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\OffsetStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\OutBuffer.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\OutBuffer.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\OutMemStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\OutMemStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\ProgressMt.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\ProgressMt.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\ProgressUtils.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\ProgressUtils.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\PropId.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\RegisterArc.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\RegisterCodec.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\StreamBinder.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\StreamBinder.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\StreamObjects.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\StreamObjects.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\StreamUtils.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\StreamUtils.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\UniqBlocks.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\UniqBlocks.h -# End Source File -# Begin Source File - -SOURCE=..\..\Common\VirtThread.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Common\VirtThread.h -# End Source File -# End Group -# Begin Group "C" - -# PROP Default_Filter "" -# Begin Group "xz" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\..\..\C\Xz.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /W4 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# ADD CPP /W4 /WX -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Xz.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzCrc64.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /W4 /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# ADD CPP /W4 /WX -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzCrc64.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzCrc64Opt.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzDec.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /W4 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# ADD CPP /W4 /WX -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzEnc.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzEnc.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\XzIn.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /W4 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# ADD CPP /W4 /WX -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# End Group -# Begin Source File - -SOURCE=..\..\..\..\C\7zBuf.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\7zBuf2.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\7zCrc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\7zCrc.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\7zStream.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\7zTypes.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Aes.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Aes.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Alloc.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Alloc.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Bcj2.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Bcj2.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Bcj2Enc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Blake2.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Blake2s.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Bra.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Bra.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Bra86.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\BraIA64.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\BwtSort.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\BwtSort.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Compiler.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\CpuArch.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\CpuArch.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Delta.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Delta.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\HuffEnc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\HuffEnc.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzFind.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzFind.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzFindMt.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzFindMt.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzHash.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Lzma2Dec.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Lzma2Dec.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Lzma2DecMt.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Lzma2DecMt.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Lzma2Enc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Lzma2Enc.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzmaDec.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzmaDec.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzmaEnc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\LzmaEnc.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\MtCoder.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\MtCoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\MtDec.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\MtDec.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd7.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd7.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd7Dec.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd7Enc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd8.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd8.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd8Dec.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Ppmd8Enc.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Precomp.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\RotateDefs.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sha1.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sha1.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sha256.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sha256.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sort.c - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# SUBTRACT CPP /YX /Yc /Yu - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Sort.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\StdAfx.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Threads.c -# SUBTRACT CPP /YX /Yc /Yu -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\C\Threads.h -# End Source File -# End Group -# Begin Group "Archive" - -# PROP Default_Filter "" -# Begin Group "7z" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\7z\7zCompressionMode.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zCompressionMode.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zDecode.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zDecode.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zEncode.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zEncode.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zExtract.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zFolderInStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zFolderInStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zHandlerOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zHeader.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zHeader.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zItem.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zOut.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zProperties.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zProperties.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zSpecStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zSpecStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zUpdate.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zUpdate.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\7z\7zUpdateItem.h -# End Source File -# End Group -# Begin Group "Rar" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Rar\Rar5Handler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Rar\Rar5Handler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Rar\RarHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Rar\RarHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Rar\RarHeader.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Rar\RarItem.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Rar\RarVol.h -# End Source File -# End Group -# Begin Group "Cab" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabBlockInStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabBlockInStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabHeader.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabHeader.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabItem.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Cab\CabRegister.cpp -# End Source File -# End Group -# Begin Group "Chm" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Chm\ChmHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Chm\ChmHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Chm\ChmIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Chm\ChmIn.h -# End Source File -# End Group -# Begin Group "Archive common" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Common\CoderMixer2.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\CoderMixer2.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\DummyOutStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\DummyOutStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\FindSignature.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\FindSignature.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\HandlerOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\HandlerOut.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\InStreamWithCRC.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\InStreamWithCRC.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\ItemNameUtils.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\ItemNameUtils.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\MultiStream.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\MultiStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\OutStreamWithCRC.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\OutStreamWithCRC.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\OutStreamWithSha1.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\OutStreamWithSha1.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\ParseProperties.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Common\ParseProperties.h -# End Source File -# End Group -# Begin Group "Iso" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoHeader.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoHeader.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoItem.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Iso\IsoRegister.cpp -# End Source File -# End Group -# Begin Group "Nsis" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisDecode.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisDecode.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Nsis\NsisRegister.cpp -# End Source File -# End Group -# Begin Group "Tar" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarHandlerOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarHeader.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarHeader.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarItem.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarOut.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarUpdate.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Tar\TarUpdate.h -# End Source File -# End Group -# Begin Group "Zip" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipAddCommon.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipAddCommon.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipCompressionMode.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipHandlerOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipHeader.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipItem.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipItem.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipOut.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipRegister.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipUpdate.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Zip\ZipUpdate.h -# End Source File -# End Group -# Begin Group "Wim" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Wim\WimHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Wim\WimHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Wim\WimHandlerOut.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Wim\WimIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Wim\WimIn.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Wim\WimRegister.cpp -# End Source File -# End Group -# Begin Group "Udf" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\Archive\Udf\UdfHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Udf\UdfHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Udf\UdfIn.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Udf\UdfIn.h -# End Source File -# End Group -# Begin Source File - -SOURCE=..\..\Archive\ApmHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ArHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ArjHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\Bz2Handler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ComHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\CpioHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\CramfsHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\DeflateProps.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\DeflateProps.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\DmgHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ElfHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ExtHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\FatHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\FlvHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\GptHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\GzHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\HandlerCont.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\HandlerCont.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\HfsHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\IArchive.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\IhexHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\LzhHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\LzmaHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\MachoHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\MbrHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\MslzHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\MubHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\NtfsHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\PeHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\PpmdHandler.cpp - -!IF "$(CFG)" == "7z - Win32 Release" - -# ADD CPP /O2 -# SUBTRACT CPP /YX /Yc /Yu - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\QcowHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\RpmHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\SplitHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\SquashfsHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\SwfHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\UefiHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\VdiHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\VhdHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\VmdkHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\XarHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\XzHandler.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\XzHandler.h -# End Source File -# Begin Source File - -SOURCE=..\..\Archive\ZHandler.cpp -# End Source File -# End Group -# Begin Group "7zip" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\ICoder.h -# End Source File -# Begin Source File - -SOURCE=..\..\IDecl.h -# End Source File -# Begin Source File - -SOURCE=..\..\IPassword.h -# End Source File -# Begin Source File - -SOURCE=..\..\IProgress.h -# End Source File -# Begin Source File - -SOURCE=..\..\IStream.h -# End Source File -# Begin Source File - -SOURCE=..\..\PropID.h -# End Source File -# End Group -# Begin Group "Windows" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\..\Windows\Defs.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileDir.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileDir.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileFind.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileFind.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileIO.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileIO.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileName.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\FileName.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\PropVariant.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\PropVariant.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\PropVariantUtils.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\PropVariantUtils.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\Synchronization.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\Synchronization.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\System.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\System.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\Thread.h -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\TimeUtils.cpp -# End Source File -# Begin Source File - -SOURCE=..\..\..\Windows\TimeUtils.h -# End Source File -# End Group -# Begin Group "Asm" - -# PROP Default_Filter "" -# Begin Source File - -SOURCE=..\..\..\..\Asm\x86\7zAsm.asm -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\Asm\x86\7zCrcOpt.asm - -!IF "$(CFG)" == "7z - Win32 Release" - -# Begin Custom Build -OutDir=.\Release -InputPath=..\..\..\..\Asm\x86\7zCrcOpt.asm -InputName=7zCrcOpt - -"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" - ml.exe -c -Fo$(OutDir)\$(InputName).obj $(InputPath) - -# End Custom Build - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# Begin Custom Build -OutDir=.\Debug -InputPath=..\..\..\..\Asm\x86\7zCrcOpt.asm -InputName=7zCrcOpt - -"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" - ml.exe -c -omf -Fo$(OutDir)\$(InputName).obj $(InputPath) - -# End Custom Build - -!ENDIF - -# End Source File -# Begin Source File - -SOURCE=..\..\..\..\Asm\x86\AesOpt.asm - -!IF "$(CFG)" == "7z - Win32 Release" - -# PROP Ignore_Default_Tool 1 -# Begin Custom Build -OutDir=.\Release -InputPath=..\..\..\..\Asm\x86\AesOpt.asm -InputName=AesOpt - -"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" - ml.exe -c -Fo$(OutDir)\$(InputName).obj $(InputPath) - -# End Custom Build - -!ELSEIF "$(CFG)" == "7z - Win32 Debug" - -# PROP Ignore_Default_Tool 1 -# Begin Custom Build -OutDir=.\Debug -InputPath=..\..\..\..\Asm\x86\AesOpt.asm -InputName=AesOpt - -"$(OutDir)\$(InputName).obj" : $(SOURCE) "$(INTDIR)" "$(OUTDIR)" - ml.exe -c -omf -WX -W3 -Fo$(OutDir)\$(InputName).obj $(InputPath) - -# End Custom Build - -!ENDIF - -# End Source File -# End Group -# End Target -# End Project diff --git a/CPP/7zip/Bundles/Format7zFO/Format7z.dsw b/CPP/7zip/Bundles/Format7zFO/Format7z.dsw deleted file mode 100644 index bb68d259..00000000 --- a/CPP/7zip/Bundles/Format7zFO/Format7z.dsw +++ /dev/null @@ -1,29 +0,0 @@ -Microsoft Developer Studio Workspace File, Format Version 6.00 -# WARNING: DO NOT EDIT OR DELETE THIS WORKSPACE FILE! - -############################################################################### - -Project: "7z"=.\Format7z.dsp - Package Owner=<4> - -Package=<5> -{{{ -}}} - -Package=<4> -{{{ -}}} - -############################################################################### - -Global: - -Package=<5> -{{{ -}}} - -Package=<3> -{{{ -}}} - -############################################################################### - diff --git a/CPP/7zip/Bundles/Format7zFO/StdAfx.cpp b/CPP/7zip/Bundles/Format7zFO/StdAfx.cpp deleted file mode 100644 index c6d3b1fa..00000000 --- a/CPP/7zip/Bundles/Format7zFO/StdAfx.cpp +++ /dev/null @@ -1,3 +0,0 @@ -// StdAfx.cpp - -#include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Format7zFO/StdAfx.h b/CPP/7zip/Bundles/Format7zFO/StdAfx.h deleted file mode 100644 index 59d9ac15..00000000 --- a/CPP/7zip/Bundles/Format7zFO/StdAfx.h +++ /dev/null @@ -1,8 +0,0 @@ -// StdAfx.h - -#ifndef __STDAFX_H -#define __STDAFX_H - -#include "../../../Common/Common.h" - -#endif diff --git a/CPP/7zip/Bundles/Format7zFO/makefile b/CPP/7zip/Bundles/Format7zFO/makefile deleted file mode 100644 index 799356c1..00000000 --- a/CPP/7zip/Bundles/Format7zFO/makefile +++ /dev/null @@ -1,134 +0,0 @@ -PROG = 7z.dll -DEF_FILE = ../../Archive/Archive2.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DEXTERNAL_CODECS -DZSTD_LEGACY_SUPPORT -DZSTD_MULTITHREAD - -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -D_7ZIP_LARGE_PAGES -!ENDIF - -!include "Arc.mak" - -COMPRESS_OBJS = $(COMPRESS_OBJS) \ - $O\CodecExports.obj \ - -AR_OBJS = $(AR_OBJS) \ - $O\ArchiveExports.obj \ - $O\DllExports2.obj \ - -COMPRESS_OBJS = $(COMPRESS_OBJS) \ - $O\BrotliDecoder.obj \ - $O\BrotliEncoder.obj \ - $O\BrotliRegister.obj \ - $O\Lz4Decoder.obj \ - $O\Lz4Encoder.obj \ - $O\Lz4Register.obj \ - $O\LizardDecoder.obj \ - $O\LizardEncoder.obj \ - $O\LizardRegister.obj \ - $O\Lz5Decoder.obj \ - $O\Lz5Encoder.obj \ - $O\Lz5Register.obj \ - $O\ZstdDecoder.obj \ - $O\ZstdEncoder.obj \ - $O\ZstdRegister.obj \ - $O\FastLzma2Register.obj \ - -BROTLI_OBJS = \ - $O/br_backward_references.obj \ - $O/br_backward_references_hq.obj \ - $O/br_bit_cost.obj \ - $O/br_bit_reader.obj \ - $O/br_block_splitter.obj \ - $O/br_brotli_bit_stream.obj \ - $O/br_cluster.obj \ - $O/br_compress_fragment.obj \ - $O/br_compress_fragment_two_pass.obj \ - $O/br_decode.obj \ - $O/br_dictionary.obj \ - $O/br_dictionary_hash.obj \ - $O/br_encode.obj \ - $O/br_encoder_dict.obj \ - $O/br_entropy_encode.obj \ - $O/br_histogram.obj \ - $O/br_huffman.obj \ - $O/br_literal_cost.obj \ - $O/br_memory.obj \ - $O/br_metablock.obj \ - $O/br_state.obj \ - $O/br_static_dict.obj \ - $O/br_transform.obj \ - $O/br_utf8_util.obj \ - -LIZARD_OBJS = \ - $O/lizard_compress.obj \ - $O/lizard_decompress.obj \ - $O/lizard_frame.obj \ - -LZ4_OBJS = \ - $O\lz4.obj \ - $O\lz4frame.obj \ - $O\lz4hc.obj \ - -LZ5_OBJS = \ - $O\lz5.obj \ - $O\lz5frame.obj \ - $O\lz5hc.obj \ - -ZSTD_OBJS = \ - $O\debug.obj \ - $O\entropy_common.obj \ - $O\error_private.obj \ - $O\fse_compress.obj \ - $O\fse_decompress.obj \ - $O\hist.obj \ - $O\huf_compress.obj \ - $O\huf_decompress.obj \ - $O\pool.obj \ - $O\threading.obj \ - $O\xxhash.obj \ - $O\zstd_common.obj \ - $O\zstd_compress.obj \ - $O\zstd_decompress.obj \ - $O\zstd_double_fast.obj \ - $O\zstd_fast.obj \ - $O\zstd_lazy.obj \ - $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ - $O\zstd_opt.obj \ - $O\zstd_v01.obj \ - $O\zstd_v02.obj \ - $O\zstd_v03.obj \ - $O\zstd_v04.obj \ - $O\zstd_v05.obj \ - $O\zstd_v06.obj \ - $O\zstd_v07.obj \ - -ZSTDMT_OBJS = \ - $O\brotli-mt_common.obj \ - $O\brotli-mt_compress.obj \ - $O\brotli-mt_decompress.obj \ - $O\lizard-mt_common.obj \ - $O\lizard-mt_compress.obj \ - $O\lizard-mt_decompress.obj \ - $O\lz4-mt_common.obj \ - $O\lz4-mt_compress.obj \ - $O\lz4-mt_decompress.obj \ - $O\lz5-mt_common.obj \ - $O\lz5-mt_compress.obj \ - $O\lz5-mt_decompress.obj \ - $O\zstd-mt_threading.obj \ - -FASTLZMA2_OBJS = \ - $O\dict_buffer.obj \ - $O\fl2_common.obj \ - $O\fl2_compress.obj \ - $O\fl2_pool.obj \ - $O\fl2_threading.obj \ - $O\lzma2_enc.obj \ - $O\radix_bitpack.obj \ - $O\radix_mf.obj \ - $O\radix_struct.obj \ - $O\range_enc.obj \ - $O\util.obj \ - -!include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zFO/resource.rc b/CPP/7zip/Bundles/Format7zFO/resource.rc deleted file mode 100644 index e7c4d1b2..00000000 --- a/CPP/7zip/Bundles/Format7zFO/resource.rc +++ /dev/null @@ -1,40 +0,0 @@ -#include "../../MyVersionInfo.rc" - -MY_VERSION_INFO_DLL("7z Plugin" , "7z") - - -0 ICON "../../Archive/Icons/7z.ico" -1 ICON "../../Archive/Icons/zip.ico" -2 ICON "../../Archive/Icons/bz2.ico" -3 ICON "../../Archive/Icons/rar.ico" -4 ICON "../../Archive/Icons/arj.ico" -5 ICON "../../Archive/Icons/z.ico" -6 ICON "../../Archive/Icons/lzh.ico" -7 ICON "../../Archive/Icons/cab.ico" -8 ICON "../../Archive/Icons/iso.ico" -9 ICON "../../Archive/Icons/split.ico" -10 ICON "../../Archive/Icons/rpm.ico" -11 ICON "../../Archive/Icons/deb.ico" -12 ICON "../../Archive/Icons/cpio.ico" -13 ICON "../../Archive/Icons/tar.ico" -14 ICON "../../Archive/Icons/gz.ico" -15 ICON "../../Archive/Icons/wim.ico" -16 ICON "../../Archive/Icons/lzma.ico" -17 ICON "../../Archive/Icons/dmg.ico" -18 ICON "../../Archive/Icons/hfs.ico" -19 ICON "../../Archive/Icons/xar.ico" -20 ICON "../../Archive/Icons/vhd.ico" -21 ICON "../../Archive/Icons/fat.ico" -22 ICON "../../Archive/Icons/ntfs.ico" -23 ICON "../../Archive/Icons/xz.ico" -24 ICON "../../Archive/Icons/squashfs.ico" -25 ICON "../../Archive/Icons/zst.ico" -26 ICON "../../Archive/Icons/lz4.ico" -27 ICON "../../Archive/Icons/lz5.ico" -28 ICON "../../Archive/Icons/liz.ico" - - -STRINGTABLE -BEGIN - 100 "7z:0 zip:1 rar:3 001:9 cab:7 iso:8 xz:23 txz:23 lzma:16 tar:13 cpio:12 bz2:2 bzip2:2 tbz2:2 tbz:2 gz:14 gzip:14 tgz:14 tpz:14 z:5 taz:5 lz:16 tlz:16 liz:28 lz4:26 lz5:27 lzh:6 lha:6 rpm:10 deb:11 arj:4 vhd:20 wim:15 swm:15 fat:21 ntfs:22 dmg:17 hfs:18 xar:19 squashfs:24 zst:25" -END diff --git a/CPP/7zip/Bundles/Format7zUSB/StdAfx.cpp b/CPP/7zip/Bundles/Format7zUSB/StdAfx.cpp deleted file mode 100644 index c6d3b1fa..00000000 --- a/CPP/7zip/Bundles/Format7zUSB/StdAfx.cpp +++ /dev/null @@ -1,3 +0,0 @@ -// StdAfx.cpp - -#include "StdAfx.h" diff --git a/CPP/7zip/Bundles/Format7zUSB/StdAfx.h b/CPP/7zip/Bundles/Format7zUSB/StdAfx.h deleted file mode 100644 index 59d9ac15..00000000 --- a/CPP/7zip/Bundles/Format7zUSB/StdAfx.h +++ /dev/null @@ -1,8 +0,0 @@ -// StdAfx.h - -#ifndef __STDAFX_H -#define __STDAFX_H - -#include "../../../Common/Common.h" - -#endif diff --git a/CPP/7zip/Bundles/Format7zUSB/makefile b/CPP/7zip/Bundles/Format7zUSB/makefile deleted file mode 100644 index 68301d24..00000000 --- a/CPP/7zip/Bundles/Format7zUSB/makefile +++ /dev/null @@ -1,251 +0,0 @@ -PROG = 7zu.dll -DEF_FILE = ../../Archive/Archive2.def -CFLAGS = $(CFLAGS) -DNEED_7ZIP_GUID -DZSTD_MULTITHREAD - -COMMON_OBJS = \ - $O\CRC.obj \ - $O\CrcReg.obj \ - $O\IntToString.obj \ - $O\NewHandler.obj \ - $O\MyString.obj \ - $O\Sha256Reg.obj \ - $O\StringConvert.obj \ - $O\StringToInt.obj \ - $O\MyVector.obj \ - $O\Wildcard.obj \ - -WIN_OBJS = \ - $O\FileDir.obj \ - $O\FileFind.obj \ - $O\FileIO.obj \ - $O\FileName.obj \ - $O\PropVariant.obj \ - $O\Synchronization.obj \ - $O\System.obj \ - -7ZIP_COMMON_OBJS = \ - $O\CreateCoder.obj \ - $O\CWrappers.obj \ - $O\InBuffer.obj \ - $O\InOutTempBuffer.obj \ - $O\FilterCoder.obj \ - $O\LimitedStreams.obj \ - $O\MethodId.obj \ - $O\MethodProps.obj \ - $O\OutBuffer.obj \ - $O\ProgressUtils.obj \ - $O\PropId.obj \ - $O\StreamBinder.obj \ - $O\StreamObjects.obj \ - $O\StreamUtils.obj \ - $O\UniqBlocks.obj \ - $O\VirtThread.obj \ - -AR_OBJS = \ - $O\ArchiveExports.obj \ - $O\DllExports2.obj \ - -AR_COMMON_OBJS = \ - $O\CoderMixer2.obj \ - $O\HandlerOut.obj \ - $O\InStreamWithCRC.obj \ - $O\ItemNameUtils.obj \ - $O\OutStreamWithCRC.obj \ - $O\ParseProperties.obj \ - - -7Z_OBJS = \ - $O\7zCompressionMode.obj \ - $O\7zDecode.obj \ - $O\7zEncode.obj \ - $O\7zExtract.obj \ - $O\7zFolderInStream.obj \ - $O\7zHandler.obj \ - $O\7zHandlerOut.obj \ - $O\7zHeader.obj \ - $O\7zIn.obj \ - $O\7zOut.obj \ - $O\7zProperties.obj \ - $O\7zSpecStream.obj \ - $O\7zUpdate.obj \ - $O\7zRegister.obj \ - - -COMPRESS_OBJS = \ - $O\CodecExports.obj \ - $O\Bcj2Coder.obj \ - $O\Bcj2Register.obj \ - $O\BcjCoder.obj \ - $O\BcjRegister.obj \ - $O\BitlDecoder.obj \ - $O\BranchMisc.obj \ - $O\BranchRegister.obj \ - $O\ByteSwap.obj \ - $O\BZip2Crc.obj \ - $O\BZip2Decoder.obj \ - $O\BZip2Encoder.obj \ - $O\BZip2Register.obj \ - $O\CopyCoder.obj \ - $O\CopyRegister.obj \ - $O\DeflateDecoder.obj \ - $O\DeflateEncoder.obj \ - $O\DeflateRegister.obj \ - $O\DeltaFilter.obj \ - $O\Lzma2Decoder.obj \ - $O\Lzma2Encoder.obj \ - $O\Lzma2Register.obj \ - $O\LzmaDecoder.obj \ - $O\LzmaEncoder.obj \ - $O\LzmaRegister.obj \ - $O\LzOutWindow.obj \ - $O\PpmdDecoder.obj \ - $O\PpmdEncoder.obj \ - $O\PpmdRegister.obj \ - -CRYPTO_OBJS = \ - $O\7zAes.obj \ - $O\7zAesRegister.obj \ - $O\MyAes.obj \ - $O\MyAesReg.obj \ - $O\RandGen.obj \ - -C_OBJS = \ - $O\Alloc.obj \ - $O\Bcj2.obj \ - $O\Bcj2Enc.obj \ - $O\Bra.obj \ - $O\Bra86.obj \ - $O\BraIA64.obj \ - $O\BwtSort.obj \ - $O\CpuArch.obj \ - $O\Delta.obj \ - $O\HuffEnc.obj \ - $O\LzFind.obj \ - $O\LzFindMt.obj \ - $O\Lzma2Dec.obj \ - $O\Lzma2DecMt.obj \ - $O\Lzma2Enc.obj \ - $O\LzmaDec.obj \ - $O\LzmaEnc.obj \ - $O\MtCoder.obj \ - $O\MtDec.obj \ - $O\Ppmd7.obj \ - $O\Ppmd7Dec.obj \ - $O\Ppmd7Enc.obj \ - $O\Sha256.obj \ - $O\Sort.obj \ - $O\Threads.obj \ - -!include "../../Aes.mak" -!include "../../Crc.mak" - -COMPRESS_OBJS = $(COMPRESS_OBJS) \ - $O\BrotliDecoder.obj \ - $O\BrotliEncoder.obj \ - $O\BrotliRegister.obj \ - $O\Lz4Decoder.obj \ - $O\Lz4Encoder.obj \ - $O\Lz4Register.obj \ - $O\LizardDecoder.obj \ - $O\LizardEncoder.obj \ - $O\LizardRegister.obj \ - $O\Lz5Decoder.obj \ - $O\Lz5Encoder.obj \ - $O\Lz5Register.obj \ - $O\ZstdDecoder.obj \ - $O\ZstdEncoder.obj \ - $O\ZstdRegister.obj \ - $O\FastLzma2Register.obj \ - -BROTLI_OBJS = \ - $O/br_backward_references.obj \ - $O/br_backward_references_hq.obj \ - $O/br_bit_cost.obj \ - $O/br_bit_reader.obj \ - $O/br_block_splitter.obj \ - $O/br_brotli_bit_stream.obj \ - $O/br_cluster.obj \ - $O/br_compress_fragment.obj \ - $O/br_compress_fragment_two_pass.obj \ - $O/br_decode.obj \ - $O/br_dictionary.obj \ - $O/br_dictionary_hash.obj \ - $O/br_encode.obj \ - $O/br_encoder_dict.obj \ - $O/br_entropy_encode.obj \ - $O/br_histogram.obj \ - $O/br_huffman.obj \ - $O/br_literal_cost.obj \ - $O/br_memory.obj \ - $O/br_metablock.obj \ - $O/br_state.obj \ - $O/br_static_dict.obj \ - $O/br_transform.obj \ - $O/br_utf8_util.obj \ - -LIZARD_OBJS = \ - $O/lizard_compress.obj \ - $O/lizard_decompress.obj \ - $O/lizard_frame.obj \ - -LZ4_OBJS = \ - $O\lz4.obj \ - $O\lz4frame.obj \ - $O\lz4hc.obj \ - -LZ5_OBJS = \ - $O\lz5.obj \ - $O\lz5frame.obj \ - $O\lz5hc.obj \ - -ZSTD_OBJS = \ - $O\debug.obj \ - $O\entropy_common.obj \ - $O\error_private.obj \ - $O\fse_compress.obj \ - $O\fse_decompress.obj \ - $O\hist.obj \ - $O\huf_compress.obj \ - $O\huf_decompress.obj \ - $O\pool.obj \ - $O\threading.obj \ - $O\xxhash.obj \ - $O\zstd_common.obj \ - $O\zstd_compress.obj \ - $O\zstd_decompress.obj \ - $O\zstd_double_fast.obj \ - $O\zstd_fast.obj \ - $O\zstd_lazy.obj \ - $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ - $O\zstd_opt.obj \ - -ZSTDMT_OBJS = \ - $O\brotli-mt_common.obj \ - $O\brotli-mt_compress.obj \ - $O\brotli-mt_decompress.obj \ - $O\lizard-mt_common.obj \ - $O\lizard-mt_compress.obj \ - $O\lizard-mt_decompress.obj \ - $O\lz4-mt_common.obj \ - $O\lz4-mt_compress.obj \ - $O\lz4-mt_decompress.obj \ - $O\lz5-mt_common.obj \ - $O\lz5-mt_compress.obj \ - $O\lz5-mt_decompress.obj \ - $O\zstd-mt_threading.obj \ - -FASTLZMA2_OBJS = \ - $O\dict_buffer.obj \ - $O\fl2_common.obj \ - $O\fl2_compress.obj \ - $O\fl2_pool.obj \ - $O\fl2_threading.obj \ - $O\lzma2_enc.obj \ - $O\radix_bitpack.obj \ - $O\radix_mf.obj \ - $O\radix_struct.obj \ - $O\range_enc.obj \ - $O\util.obj \ - -!include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/Format7zUSB/resource.rc b/CPP/7zip/Bundles/Format7zUSB/resource.rc deleted file mode 100644 index 53d6be06..00000000 --- a/CPP/7zip/Bundles/Format7zUSB/resource.rc +++ /dev/null @@ -1,5 +0,0 @@ -#include "../../MyVersionInfo.rc" - -MY_VERSION_INFO_DLL("7z Standalone Plugin", "7za") - -101 ICON "../../Archive/Icons/7z.ico" diff --git a/CPP/7zip/Bundles/LzmaCon/makefile b/CPP/7zip/Bundles/LzmaCon/makefile index 26097630..5a50808f 100644 --- a/CPP/7zip/Bundles/LzmaCon/makefile +++ b/CPP/7zip/Bundles/LzmaCon/makefile @@ -1,6 +1,5 @@ PROG = lzma.exe MY_CONSOLE = 1 -CFLAGS = $(CFLAGS) CURRENT_OBJS = \ $O\LzmaAlone.obj \ @@ -55,5 +54,6 @@ C_OBJS = \ $O\Threads.obj \ !include "../../Crc.mak" +!include "../../LzmaDec.mak" !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXCon/makefile b/CPP/7zip/Bundles/SFXCon/makefile index e6adea09..c785ceb9 100644 --- a/CPP/7zip/Bundles/SFXCon/makefile +++ b/CPP/7zip/Bundles/SFXCon/makefile @@ -6,7 +6,6 @@ CFLAGS = $(CFLAGS) \ -DEXTRACT_ONLY \ -DNO_READ_FROM_CODER \ -D_SFX \ - -D_CONSOLE \ CURRENT_OBJS = \ $O\SfxCon.obj \ @@ -134,29 +133,20 @@ C_OBJS = \ COMPRESS_OBJS = $(COMPRESS_OBJS) \ $O\ZstdDecoder.obj \ - $O\ZstdEncoder.obj \ $O\ZstdRegister.obj \ ZSTD_OBJS = \ $O\debug.obj \ $O\entropy_common.obj \ $O\error_private.obj \ - $O\fse_compress.obj \ $O\fse_decompress.obj \ - $O\hist.obj \ - $O\huf_compress.obj \ $O\huf_decompress.obj \ $O\pool.obj \ $O\threading.obj \ $O\xxhash.obj \ $O\zstd_common.obj \ - $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ - $O\zstd_double_fast.obj \ - $O\zstd_fast.obj \ - $O\zstd_lazy.obj \ - $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ - $O\zstd_opt.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/Bundles/SFXWin/makefile b/CPP/7zip/Bundles/SFXWin/makefile index 8bc02a3f..8a2145bf 100644 --- a/CPP/7zip/Bundles/SFXWin/makefile +++ b/CPP/7zip/Bundles/SFXWin/makefile @@ -152,29 +152,21 @@ C_OBJS = \ COMPRESS_OBJS = $(COMPRESS_OBJS) \ $O\ZstdDecoder.obj \ - $O\ZstdEncoder.obj \ $O\ZstdRegister.obj \ ZSTD_OBJS = \ $O\debug.obj \ $O\entropy_common.obj \ $O\error_private.obj \ - $O\fse_compress.obj \ $O\fse_decompress.obj \ - $O\hist.obj \ - $O\huf_compress.obj \ $O\huf_decompress.obj \ $O\pool.obj \ $O\threading.obj \ $O\xxhash.obj \ $O\zstd_common.obj \ - $O\zstd_compress.obj \ + $O\zstd_ddict.obj \ + $O\zstd_decompress_block.obj \ $O\zstd_decompress.obj \ - $O\zstd_double_fast.obj \ - $O\zstd_fast.obj \ - $O\zstd_lazy.obj \ - $O\zstd_ldm.obj \ - $O\zstdmt_compress.obj \ - $O\zstd_opt.obj \ + !include "../../7zip.mak" diff --git a/CPP/7zip/Common/FilterCoder.cpp b/CPP/7zip/Common/FilterCoder.cpp index 275c60d4..d5c7ff0e 100644 --- a/CPP/7zip/Common/FilterCoder.cpp +++ b/CPP/7zip/Common/FilterCoder.cpp @@ -7,6 +7,23 @@ #include "FilterCoder.h" #include "StreamUtils.h" +#ifdef _WIN32 + #define alignedMidBuffer_Alloc g_MidAlloc +#else + #define alignedMidBuffer_Alloc g_AlignedAlloc +#endif + +CAlignedMidBuffer::~CAlignedMidBuffer() +{ + ISzAlloc_Free(&alignedMidBuffer_Alloc, _buf); +} + +void CAlignedMidBuffer::AllocAligned(size_t size) +{ + ISzAlloc_Free(&alignedMidBuffer_Alloc, _buf); + _buf = (Byte *)ISzAlloc_Alloc(&alignedMidBuffer_Alloc, size); +} + /* AES filters need 16-bytes alignment for HARDWARE-AES instructions. So we call IFilter::Filter(, size), where (size != 16 * N) only for last data block. @@ -36,7 +53,7 @@ HRESULT CFilterCoder::Alloc() size = kMinSize; if (!_buf || _bufSize != size) { - AllocAlignedMask(size, 16 - 1); + AllocAligned(size); if (!_buf) return E_OUTOFMEMORY; _bufSize = size; diff --git a/CPP/7zip/Common/FilterCoder.h b/CPP/7zip/Common/FilterCoder.h index 0e2f84f1..bde0e2bb 100644 --- a/CPP/7zip/Common/FilterCoder.h +++ b/CPP/7zip/Common/FilterCoder.h @@ -19,41 +19,11 @@ struct CAlignedMidBuffer { - #ifdef _WIN32 - Byte *_buf; CAlignedMidBuffer(): _buf(NULL) {} - ~CAlignedMidBuffer() { ::MidFree(_buf); } - - void AllocAlignedMask(size_t size, size_t) - { - ::MidFree(_buf); - _buf = (Byte *)::MidAlloc(size); - } - - #else - - Byte *_bufBase; - Byte *_buf; - - CAlignedMidBuffer(): _bufBase(NULL), _buf(NULL) {} - ~CAlignedMidBuffer() { ::MidFree(_bufBase); } - - void AllocAlignedMask(size_t size, size_t alignMask) - { - ::MidFree(_bufBase); - _buf = NULL; - _bufBase = (Byte *)::MidAlloc(size + alignMask); - - if (_bufBase) - { - // _buf = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - _buf = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - } - } - - #endif + ~CAlignedMidBuffer(); + void AllocAligned(size_t size); }; class CFilterCoder: diff --git a/CPP/7zip/Common/MethodProps.cpp b/CPP/7zip/Common/MethodProps.cpp index 2134462c..2935e6da 100644 --- a/CPP/7zip/Common/MethodProps.cpp +++ b/CPP/7zip/Common/MethodProps.cpp @@ -279,7 +279,21 @@ static const CNameToPropID g_NameToPropID[] = { VT_UI4, "b" }, { VT_UI4, "check" }, { VT_BSTR, "filter" }, - { VT_UI8, "memuse" } + { VT_UI8, "memuse" }, + { VT_UI4, "strat" }, + { VT_UI4, "fast" }, + { VT_UI4, "long" }, + { VT_UI4, "wlog" }, + { VT_UI4, "hlog" }, + { VT_UI4, "clog" }, + { VT_UI4, "slog" }, + { VT_UI4, "slen" }, + { VT_UI4, "tlen" }, + { VT_UI4, "ovlog" }, + { VT_UI4, "ldmhlog" }, + { VT_UI4, "ldmslen" }, + { VT_UI4, "ldmblog" }, + { VT_UI4, "ldmhevery" } }; static int FindPropIdExact(const UString &name) diff --git a/CPP/7zip/Compress/DeflateDecoder.cpp b/CPP/7zip/Compress/DeflateDecoder.cpp index e456f82b..d77fd2b2 100644 --- a/CPP/7zip/Compress/DeflateDecoder.cpp +++ b/CPP/7zip/Compress/DeflateDecoder.cpp @@ -446,6 +446,14 @@ void CCoder::SetOutStreamSizeResume(const UInt64 *outSize) STDMETHODIMP CCoder::SetOutStreamSize(const UInt64 *outSize) { + /* + 18.06: + We want to support GetInputProcessedSize() before CCoder::Read() + So we call m_InBitStream.Init() even before buffer allocations + m_InBitStream.Init() just sets variables to default values + But later we will call m_InBitStream.Init() again with real buffer pointers + */ + m_InBitStream.Init(); _needInitInStream = true; SetOutStreamSizeResume(outSize); return S_OK; diff --git a/CPP/7zip/Compress/Lz4Decoder.cpp b/CPP/7zip/Compress/Lz4Decoder.cpp index 1631e1c8..cb88d086 100644 --- a/CPP/7zip/Compress/Lz4Decoder.cpp +++ b/CPP/7zip/Compress/Lz4Decoder.cpp @@ -87,12 +87,16 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) { DProps *pProps = (DProps *)prop; - if (size != sizeof(DProps)) + switch (size) { + case 3: + memcpy(&_props, pProps, 3); + return S_OK; + case 5: + memcpy(&_props, pProps, 5); + return S_OK; + default: return E_NOTIMPL; - - memcpy(&_props, pProps, sizeof (DProps)); - - return S_OK; + } } STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) diff --git a/CPP/7zip/Compress/Lz5Decoder.cpp b/CPP/7zip/Compress/Lz5Decoder.cpp index 436272ae..5ec807cb 100644 --- a/CPP/7zip/Compress/Lz5Decoder.cpp +++ b/CPP/7zip/Compress/Lz5Decoder.cpp @@ -87,12 +87,16 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) { DProps *pProps = (DProps *)prop; - if (size != sizeof(DProps)) + switch (size) { + case 3: + memcpy(&_props, pProps, 3); + return S_OK; + case 5: + memcpy(&_props, pProps, 5); + return S_OK; + default: return E_NOTIMPL; - - memcpy(&_props, pProps, sizeof (DProps)); - - return S_OK; + } } STDMETHODIMP CDecoder::SetNumberOfThreads(UInt32 numThreads) diff --git a/CPP/7zip/Compress/LzmsDecoder.cpp b/CPP/7zip/Compress/LzmsDecoder.cpp index fd1fac0b..f67548b3 100644 --- a/CPP/7zip/Compress/LzmsDecoder.cpp +++ b/CPP/7zip/Compress/LzmsDecoder.cpp @@ -517,9 +517,9 @@ HRESULT CDecoder::CodeReal(const Byte *in, size_t inSize, Byte *_win, size_t out if (len > outSize - _pos) return S_FALSE; - if (dist > _pos) - return S_FALSE; size_t span = (size_t)1 << power; + if ((UInt64)dist + span > _pos) + return S_FALSE; Byte *dest = _win + _pos - span; const Byte *src = dest - dist; _pos += len; diff --git a/CPP/7zip/Compress/Rar3Decoder.cpp b/CPP/7zip/Compress/Rar3Decoder.cpp index a37f9eef..7b1b31fb 100644 --- a/CPP/7zip/Compress/Rar3Decoder.cpp +++ b/CPP/7zip/Compress/Rar3Decoder.cpp @@ -148,7 +148,8 @@ void CDecoder::ExecuteFilter(unsigned tempFilterIndex, NVm::CBlockRef &outBlockR if (!_vm.Execute(filter, tempFilter, outBlockRef, filter->GlobalData)) _unsupportedFilter = true; delete tempFilter; - _tempFilters[tempFilterIndex] = 0; + _tempFilters[tempFilterIndex] = NULL; + _numEmptyTempFilters++; } HRESULT CDecoder::WriteBuf() @@ -225,6 +226,7 @@ HRESULT CDecoder::WriteBuf() void CDecoder::InitFilters() { _lastFilter = 0; + _numEmptyTempFilters = 0; unsigned i; for (i = 0; i < _tempFilters.Size(); i++) delete _tempFilters[i]; @@ -274,24 +276,27 @@ bool CDecoder::AddVmCode(UInt32 firstByte, UInt32 codeSize) filter->ExecCount++; } - unsigned numEmptyItems = 0; + if (_numEmptyTempFilters != 0) { - FOR_VECTOR (i, _tempFilters) + unsigned num = _tempFilters.Size(); + CTempFilter **tempFilters = &_tempFilters.Front(); + + unsigned w = 0; + for (unsigned i = 0; i < num; i++) { - _tempFilters[i - numEmptyItems] = _tempFilters[i]; - if (!_tempFilters[i]) - numEmptyItems++; - if (numEmptyItems != 0) - _tempFilters[i] = NULL; + CTempFilter *tf = tempFilters[i]; + if (tf) + tempFilters[w++] = tf; } + + _tempFilters.DeleteFrom(w); + _numEmptyTempFilters = 0; } - if (numEmptyItems == 0) - { - _tempFilters.Add(NULL); - numEmptyItems = 1; - } + + if (_tempFilters.Size() > MAX_UNPACK_FILTERS) + return false; CTempFilter *tempFilter = new CTempFilter; - _tempFilters[_tempFilters.Size() - numEmptyItems] = tempFilter; + _tempFilters.Add(tempFilter); tempFilter->FilterIndex = filterIndex; UInt32 blockStart = inp.ReadEncodedUInt32(); diff --git a/CPP/7zip/Compress/Rar3Decoder.h b/CPP/7zip/Compress/Rar3Decoder.h index 50462987..68d79dbd 100644 --- a/CPP/7zip/Compress/Rar3Decoder.h +++ b/CPP/7zip/Compress/Rar3Decoder.h @@ -189,6 +189,7 @@ class CDecoder: NVm::CVm _vm; CRecordVector _filters; CRecordVector _tempFilters; + unsigned _numEmptyTempFilters; UInt32 _lastFilter; bool _isSolid; diff --git a/CPP/7zip/Compress/XpressDecoder.cpp b/CPP/7zip/Compress/XpressDecoder.cpp index 4c48ce3f..c143ccf3 100644 --- a/CPP/7zip/Compress/XpressDecoder.cpp +++ b/CPP/7zip/Compress/XpressDecoder.cpp @@ -33,10 +33,11 @@ struct CBitStream bs.Value = (bs.Value << 16) | GetUi16(in); \ in += 2; bs.BitPos += 16; } -const unsigned kNumHuffBits = 15; -const unsigned kNumLenSlots = 16; -const unsigned kNumPosSlots = 16; -const unsigned kNumSyms = 256 + kNumPosSlots * kNumLenSlots; +static const unsigned kNumHuffBits = 15; +static const unsigned kNumLenBits = 4; +static const unsigned kLenMask = (1 << kNumLenBits) - 1; +static const unsigned kNumPosSlots = 16; +static const unsigned kNumSyms = 256 + (kNumPosSlots << kNumLenBits); HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) { @@ -83,10 +84,10 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) else { sym -= 256; - UInt32 dist = sym / kNumLenSlots; - UInt32 len = sym & (kNumLenSlots - 1); + UInt32 dist = sym >> kNumLenBits; + UInt32 len = sym & kLenMask; - if (len == kNumLenSlots - 1) + if (len == kLenMask) { if (in > lim) return S_FALSE; @@ -99,7 +100,7 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) in += 2; } else - len += kNumLenSlots - 1; + len += kLenMask; } bs.BitPos -= dist; @@ -108,7 +109,7 @@ HRESULT Decode(const Byte *in, size_t inSize, Byte *out, size_t outSize) BIT_STREAM_NORMALIZE - if (len > outSize - pos) + if (len + 3 > outSize - pos) return S_FALSE; if (dist > pos) return S_FALSE; diff --git a/CPP/7zip/Compress/ZstdDecoder.cpp b/CPP/7zip/Compress/ZstdDecoder.cpp index baccadf0..6430c817 100644 --- a/CPP/7zip/Compress/ZstdDecoder.cpp +++ b/CPP/7zip/Compress/ZstdDecoder.cpp @@ -37,10 +37,17 @@ STDMETHODIMP CDecoder::SetDecoderProperties2(const Byte * prop, UInt32 size) { DProps *pProps = (DProps *)prop; +#if DEBUG + printf("prop size =%u\n", size); + fflush(stdout); +#endif + switch (size) { case 3: + memcpy(&_props, pProps, 3); + return S_OK; case 5: - memcpy(&_props, pProps, sizeof (DProps)); + memcpy(&_props, pProps, 5); return S_OK; default: return E_NOTIMPL; @@ -84,10 +91,6 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, ZSTD_resetDStream(_ctx); } - zIn.src = _srcBuf; - zIn.size = _srcBufSize; - zIn.pos = 0; - zOut.dst = _dstBuf; srcBufLen = _srcBufSize; @@ -95,6 +98,10 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, RINOK(ReadStream(inStream, _srcBuf, &srcBufLen)); _processedIn += srcBufLen; + zIn.src = _srcBuf; + zIn.size = srcBufLen; + zIn.pos = 0; + /* Main decompression Loop */ for (;;) { for (;;) { @@ -103,16 +110,16 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, zOut.pos = 0; result = ZSTD_decompressStream(_ctx, &zOut, &zIn); - if (ZSTD_isError(result)) { + if (ZSTD_isError(result)) return E_FAIL; - } #if DEBUG - printf("res =%u\n", (unsigned)result); - printf("zIn.size =%u\n", (unsigned)zIn.size); - printf("zIn.pos =%u\n", (unsigned)zIn.pos); - printf("zOut.size =%u\n", (unsigned)zOut.size); - printf("zOut.pos =%u\n", (unsigned)zOut.pos); + printf("res = %u\n", (unsigned)result); + printf("zIn.size = %u\n", (unsigned)zIn.size); + printf("zIn.pos = %u\n", (unsigned)zIn.pos); + printf("zOut.size = %u\n", (unsigned)zOut.size); + printf("zOut.pos = %u\n", (unsigned)zOut.pos); + printf("---------------------\n"); fflush(stdout); #endif @@ -120,13 +127,12 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, if (zOut.pos) { RINOK(WriteStream(outStream, _dstBuf, zOut.pos)); _processedOut += zOut.pos; - RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); + if (progress) + { + RINOK(progress->SetRatioInfo(&_processedIn, &_processedOut)); + } } - /* one more round */ - if ((zIn.pos == zIn.size) && (result == 1) && zOut.pos) - continue; - /* finished with buffer */ if (zIn.pos == zIn.size) break; @@ -136,6 +142,11 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, result = ZSTD_resetDStream(_ctx); if (ZSTD_isError(result)) return E_FAIL; + + /* end of frame, but more data */ + if (zIn.pos < zIn.size) + continue; + /* read next input, or eof */ break; } @@ -145,6 +156,9 @@ HRESULT CDecoder::CodeSpec(ISequentialInStream * inStream, srcBufLen = _srcBufSize; RINOK(ReadStream(inStream, _srcBuf, &srcBufLen)); _processedIn += srcBufLen; +#if DEBUG + printf("READ = %u\n", (unsigned)srcBufLen); +#endif /* finished */ if (srcBufLen == 0) diff --git a/CPP/7zip/Compress/ZstdEncoder.cpp b/CPP/7zip/Compress/ZstdEncoder.cpp index ff74f426..6ef8a0fb 100644 --- a/CPP/7zip/Compress/ZstdEncoder.cpp +++ b/CPP/7zip/Compress/ZstdEncoder.cpp @@ -22,7 +22,21 @@ CEncoder::CEncoder(): _dstBufSize(ZSTD_CStreamOutSize()), _processedIn(0), _processedOut(0), - _numThreads(NWindows::NSystem::GetNumberOfProcessors()) + _numThreads(NWindows::NSystem::GetNumberOfProcessors()), + _Long(-1), + _Level(ZSTD_CLEVEL_DEFAULT), + _Strategy(-1), + _WindowLog(-1), + _HashLog(-1), + _ChainLog(-1), + _SearchLog(-1), + _MinMatch(-1), + _TargetLen(-1), + _OverlapLog(-1), + _LdmHashLog(-1), + _LdmMinMatch(-1), + _LdmBucketSizeLog(-1), + _LdmHashRateLog(-1) { _props.clear(); _hMutex = CreateMutex(NULL, FALSE, NULL); @@ -49,24 +63,145 @@ STDMETHODIMP CEncoder::SetCoderProperties(const PROPID * propIDs, const PROPVARI UInt32 v = (UInt32)prop.ulVal; switch (propID) { - case NCoderPropID::kLevel: - { - if (prop.vt != VT_UI4) - return E_INVALIDARG; - - /* level 1..22 */ - _props._level = static_cast < Byte > (prop.ulVal); - Byte mylevel = static_cast < Byte > (ZSTD_LEVEL_MAX); - if (_props._level > mylevel) - _props._level = mylevel; - - break; - } case NCoderPropID::kNumThreads: { SetNumberOfThreads(v); break; } + case NCoderPropID::kStrategy: + { + if (v < 1) v = 1; + if (v > 8) v = 8; + _Strategy = v; + break; + } + case NCoderPropID::kLevel: + { + _Level = v; + if (v < 1) { + _Level = 1; + } else if ((Int32)v > ZSTD_maxCLevel()) { + _Level = ZSTD_maxCLevel(); + } + + /** + * zstd default levels: _Level => 1..ZSTD_maxCLevel() + */ + _props._level = static_cast < Byte > (_Level); + break; + } + case NCoderPropID::kFast: + { + /* like --fast in zstd cli program */ + UInt32 _Fast = v; + + if (v < 1) { + _Fast = 1; + } else if (v > 64) { + _Fast = 64; + } + + /** + * zstd fast levels: + * _Fast => 1..ZSTD_minCLevel() (_Level => _Fast + 32) + */ + _props._level = static_cast < Byte > (_Fast + 32); + + /* negative levels are the fast ones */ + _Level = _Fast * -1; + break; + } + case NCoderPropID::kLong: + { + /* like --long in zstd cli program */ + _Long = 1; + if (v == 0) { + // m0=zstd:long:tlen=x + _WindowLog = 27; + } else if (v < 10) { + _WindowLog = 10; + } else if (v > ZSTD_WINDOWLOG_MAX) { + _WindowLog = ZSTD_WINDOWLOG_MAX; + } + break; + } + case NCoderPropID::kWindowLog: + { + if (v < ZSTD_WINDOWLOG_MIN) v = ZSTD_WINDOWLOG_MIN; + if (v > ZSTD_WINDOWLOG_MAX) v = ZSTD_WINDOWLOG_MAX; + _WindowLog = v; + break; + } + case NCoderPropID::kHashLog: + { + if (v < ZSTD_HASHLOG_MIN) v = ZSTD_HASHLOG_MIN; + if (v > ZSTD_HASHLOG_MAX) v = ZSTD_HASHLOG_MAX; + _HashLog = v; + break; + } + case NCoderPropID::kChainLog: + { + if (v < ZSTD_CHAINLOG_MIN) v = ZSTD_CHAINLOG_MIN; + if (v > ZSTD_CHAINLOG_MAX) v = ZSTD_CHAINLOG_MAX; + _ChainLog = v; + break; + } + case NCoderPropID::kSearchLog: + { + if (v < ZSTD_SEARCHLOG_MIN) v = ZSTD_SEARCHLOG_MIN; + if (v > ZSTD_SEARCHLOG_MAX) v = ZSTD_SEARCHLOG_MAX; + _SearchLog = v; + break; + } + case NCoderPropID::kMinMatch: + { + if (v < ZSTD_MINMATCH_MIN) v = ZSTD_MINMATCH_MIN; + if (v > ZSTD_MINMATCH_MAX) v = ZSTD_MINMATCH_MAX; + _MinMatch = v; + break; + } + case NCoderPropID::kTargetLen: + { + if (v < ZSTD_TARGETLENGTH_MIN) v = ZSTD_TARGETLENGTH_MIN; + if (v > ZSTD_TARGETLENGTH_MAX) v = ZSTD_TARGETLENGTH_MAX; + _TargetLen = 0; + break; + } + case NCoderPropID::kOverlapLog: + { + if (v < 0) v = 0; /* no overlap */ + if (v > 9) v = 9; /* full size */ + _OverlapLog = v; + break; + } + case NCoderPropID::kLdmHashLog: + { + if (v < ZSTD_HASHLOG_MIN) v = ZSTD_HASHLOG_MIN; + if (v > ZSTD_HASHLOG_MAX) v = ZSTD_HASHLOG_MAX; + _LdmHashLog = v; + break; + } + case NCoderPropID::kLdmSearchLength: + { + if (v < ZSTD_LDM_MINMATCH_MIN) v = ZSTD_LDM_MINMATCH_MIN; + if (v > ZSTD_LDM_MINMATCH_MAX) v = ZSTD_LDM_MINMATCH_MAX; + _LdmMinMatch = v; + break; + } + case NCoderPropID::kLdmBucketSizeLog: + { + if (v < 1) v = 1; + if (v > ZSTD_LDM_BUCKETSIZELOG_MAX) v = ZSTD_LDM_BUCKETSIZELOG_MAX; + _LdmBucketSizeLog = v; + break; + } + case NCoderPropID::kLdmHashRateLog: + { + if (v < 0) v = 0; /* 0 => automatic mode */ + if (v > (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN)) v = (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN); + _LdmHashRateLog = v; + break; + } default: { break; @@ -105,20 +240,86 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, return E_OUTOFMEMORY; /* setup level */ - err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_compressionLevel, _props._level); - if (ZSTD_isError(err)) return E_FAIL; + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_compressionLevel, (UInt32)_Level); + if (ZSTD_isError(err)) return E_INVALIDARG; /* setup thread count */ - err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_nbWorkers, _numThreads); - if (ZSTD_isError(err)) return E_FAIL; + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_nbWorkers, _numThreads); + if (ZSTD_isError(err)) return E_INVALIDARG; /* set the content size flag */ - err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_contentSizeFlag, 1); - if (ZSTD_isError(err)) return E_FAIL; + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_contentSizeFlag, 1); + if (ZSTD_isError(err)) return E_INVALIDARG; - /* todo: make this optional */ - err = ZSTD_CCtx_setParameter(_ctx, ZSTD_p_enableLongDistanceMatching, 1); - if (ZSTD_isError(err)) return E_FAIL; + /* enable ldm for large windowlog values */ + if (_WindowLog > 27 && _Long == 0) + _Long = 1; + + /* set ldm */ + if (_Long != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_enableLongDistanceMatching, _Long); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_Strategy != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_strategy, _Strategy); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_WindowLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_windowLog, _WindowLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_HashLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_hashLog, _HashLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_ChainLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_chainLog, _ChainLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_SearchLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_searchLog, _SearchLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_MinMatch != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_minMatch, _MinMatch); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_TargetLen != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_targetLength, _TargetLen); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_OverlapLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_overlapLog, _OverlapLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_LdmHashLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_ldmHashLog, _LdmHashLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_LdmMinMatch != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_ldmMinMatch, _LdmMinMatch); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_LdmBucketSizeLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_ldmBucketSizeLog, _LdmBucketSizeLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } + + if (_LdmHashRateLog != -1) { + err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_ldmHashRateLog, _LdmHashRateLog); + if (ZSTD_isError(err)) return E_INVALIDARG; + } } for (;;) { @@ -151,7 +352,7 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream, inBuff.pos = 0; } - err = ZSTD_compress_generic(_ctx, &outBuff, &inBuff, ZSTD_todo); + err = ZSTD_compressStream2(_ctx, &outBuff, &inBuff, ZSTD_todo); if (ZSTD_isError(err)) return E_FAIL; #if DEBUG diff --git a/CPP/7zip/Compress/ZstdEncoder.h b/CPP/7zip/Compress/ZstdEncoder.h index 1f2c69a1..b2938fcb 100644 --- a/CPP/7zip/Compress/ZstdEncoder.h +++ b/CPP/7zip/Compress/ZstdEncoder.h @@ -51,6 +51,22 @@ class CEncoder: UInt32 _numThreads; HANDLE _hMutex; + /* zstd advanced compression options */ + Int32 _Long; + Int32 _Level; + Int32 _Strategy; + Int32 _WindowLog; + Int32 _HashLog; + Int32 _ChainLog; + Int32 _SearchLog; + Int32 _MinMatch; + Int32 _TargetLen; + Int32 _OverlapLog; + Int32 _LdmHashLog; + Int32 _LdmMinMatch; + Int32 _LdmBucketSizeLog; + Int32 _LdmHashRateLog; + public: MY_QUERYINTERFACE_BEGIN2(ICompressCoder) MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt) diff --git a/CPP/7zip/Crc.mak b/CPP/7zip/Crc.mak index 19a7f7b1..66b35c1e 100644 --- a/CPP/7zip/Crc.mak +++ b/CPP/7zip/Crc.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\7zCrc.obj -!IF "$(CPU)" == "IA64" || "$(CPU)" == "MIPS" || "$(CPU)" == "ARM" || "$(CPU)" == "ARM64" +!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/CPP/7zip/Crc64.mak b/CPP/7zip/Crc64.mak index 1ac6a0ce..6df9b406 100644 --- a/CPP/7zip/Crc64.mak +++ b/CPP/7zip/Crc64.mak @@ -1,6 +1,6 @@ C_OBJS = $(C_OBJS) \ $O\XzCrc64.obj -!IF "$(CPU)" == "IA64" || "$(CPU)" == "MIPS" || "$(CPU)" == "ARM" || "$(CPU)" == "ARM64" +!IF "$(PLATFORM)" == "ia64" || "$(PLATFORM)" == "mips" || "$(PLATFORM)" == "arm" || "$(PLATFORM)" == "arm64" C_OBJS = $(C_OBJS) \ !ELSE ASM_OBJS = $(ASM_OBJS) \ diff --git a/CPP/7zip/Crypto/7zAes.cpp b/CPP/7zip/Crypto/7zAes.cpp index f412bf9d..55e40f38 100644 --- a/CPP/7zip/Crypto/7zAes.cpp +++ b/CPP/7zip/Crypto/7zAes.cpp @@ -164,8 +164,8 @@ STDMETHODIMP CEncoder::ResetInitVector() { for (unsigned i = 0; i < sizeof(_iv); i++) _iv[i] = 0; - _ivSize = 8; - g_RandomGenerator.Generate(_iv, _ivSize); + _ivSize = 16; + MY_RAND_GEN(_iv, _ivSize); return S_OK; } diff --git a/CPP/7zip/Crypto/RandGen.cpp b/CPP/7zip/Crypto/RandGen.cpp index 542f39bd..791275f9 100644 --- a/CPP/7zip/Crypto/RandGen.cpp +++ b/CPP/7zip/Crypto/RandGen.cpp @@ -2,14 +2,44 @@ #include "StdAfx.h" +#include "RandGen.h" + +#ifndef USE_STATIC_SYSTEM_RAND + #ifndef _7ZIP_ST #include "../../Windows/Synchronization.h" #endif -#include "RandGen.h" -#ifndef _WIN32 +#ifdef _WIN32 + +#ifdef _WIN64 +#define USE_STATIC_RtlGenRandom +#endif + +#ifdef USE_STATIC_RtlGenRandom + +#include + +EXTERN_C_BEGIN +#ifndef RtlGenRandom + #define RtlGenRandom SystemFunction036 + BOOLEAN WINAPI RtlGenRandom(PVOID RandomBuffer, ULONG RandomBufferLength); +#endif +EXTERN_C_END + +#else +EXTERN_C_BEGIN +typedef BOOLEAN (WINAPI * Func_RtlGenRandom)(PVOID RandomBuffer, ULONG RandomBufferLength); +EXTERN_C_END +#endif + + +#else #include +#include +#include +#include #define USE_POSIX_TIME #define USE_POSIX_TIME2 #endif @@ -21,11 +51,9 @@ #endif #endif -// This is not very good random number generator. -// Please use it only for salt. -// First generated data block depends from timer and processID. +// The seed and first generated data block depend from processID, +// theadID, timer and system random generator, if available. // Other generated data blocks depend from previous state -// Maybe it's possible to restore original timer value from generated value. #define HASH_UPD(x) Sha256_Update(&hash, (const Byte *)&x, sizeof(x)); @@ -34,25 +62,102 @@ void CRandomGenerator::Init() CSha256 hash; Sha256_Init(&hash); + unsigned numIterations = 1000; + + { + #ifndef UNDER_CE + const unsigned kNumIterations_Small = 100; + const unsigned kBufSize = 32; + Byte buf[kBufSize]; + #endif + #ifdef _WIN32 + DWORD w = ::GetCurrentProcessId(); HASH_UPD(w); w = ::GetCurrentThreadId(); HASH_UPD(w); + + #ifdef UNDER_CE + /* + if (CeGenRandom(kBufSize, buf)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } + */ + #elif defined(USE_STATIC_RtlGenRandom) + if (RtlGenRandom(buf, kBufSize)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } #else + { + HMODULE hModule = ::LoadLibrary(TEXT("Advapi32.dll")); + if (hModule) + { + // SystemFunction036() is real name of RtlGenRandom() function + Func_RtlGenRandom my_RtlGenRandom = (Func_RtlGenRandom)GetProcAddress(hModule, "SystemFunction036"); + if (my_RtlGenRandom) + { + if (my_RtlGenRandom(buf, kBufSize)) + { + numIterations = kNumIterations_Small; + Sha256_Update(&hash, buf, kBufSize); + } + } + ::FreeLibrary(hModule); + } + } + #endif + + #else + pid_t pid = getpid(); HASH_UPD(pid); pid = getppid(); HASH_UPD(pid); + + { + int f = open("/dev/urandom", O_RDONLY); + unsigned numBytes = kBufSize; + if (f >= 0) + { + do + { + int n = read(f, buf, numBytes); + if (n <= 0) + break; + Sha256_Update(&hash, buf, n); + numBytes -= n; + } + while (numBytes); + close(f); + if (numBytes == 0) + numIterations = kNumIterations_Small; + } + } + /* + { + int n = getrandom(buf, kBufSize, 0); + if (n > 0) + { + Sha256_Update(&hash, buf, n); + if (n == kBufSize) + numIterations = kNumIterations_Small; + } + } + */ + + #endif + } + + #ifdef _DEBUG + numIterations = 2; #endif - for (unsigned i = 0; i < - #ifdef _DEBUG - 2; - #else - 1000; - #endif - i++) + do { #ifdef _WIN32 LARGE_INTEGER v; @@ -85,6 +190,8 @@ void CRandomGenerator::Init() Sha256_Update(&hash, _buff, SHA256_DIGEST_SIZE); } } + while (--numIterations); + Sha256_Final(&hash, _buff); _needInit = false; } @@ -122,3 +229,5 @@ void CRandomGenerator::Generate(Byte *data, unsigned size) } CRandomGenerator g_RandomGenerator; + +#endif diff --git a/CPP/7zip/Crypto/RandGen.h b/CPP/7zip/Crypto/RandGen.h index ff44450f..dc5a3386 100644 --- a/CPP/7zip/Crypto/RandGen.h +++ b/CPP/7zip/Crypto/RandGen.h @@ -5,6 +5,21 @@ #include "../../../C/Sha256.h" +#ifdef _WIN64 +// #define USE_STATIC_SYSTEM_RAND +#endif + +#ifdef USE_STATIC_SYSTEM_RAND + +#ifdef _WIN32 +#include +#define MY_RAND_GEN(data, size) RtlGenRandom(data, size) +#else +#define MY_RAND_GEN(data, size) getrandom(data, size, 0) +#endif + +#else + class CRandomGenerator { Byte _buff[SHA256_DIGEST_SIZE]; @@ -18,4 +33,8 @@ public: extern CRandomGenerator g_RandomGenerator; +#define MY_RAND_GEN(data, size) g_RandomGenerator.Generate(data, size) + +#endif + #endif diff --git a/CPP/7zip/Crypto/WzAes.cpp b/CPP/7zip/Crypto/WzAes.cpp index 115e0f4b..1c791957 100644 --- a/CPP/7zip/Crypto/WzAes.cpp +++ b/CPP/7zip/Crypto/WzAes.cpp @@ -96,7 +96,7 @@ STDMETHODIMP CBaseCoder::Init() HRESULT CEncoder::WriteHeader(ISequentialOutStream *outStream) { unsigned saltSize = _key.GetSaltSize(); - g_RandomGenerator.Generate(_key.Salt, saltSize); + MY_RAND_GEN(_key.Salt, saltSize); Init2(); RINOK(WriteStream(outStream, _key.Salt, saltSize)); return WriteStream(outStream, _key.PwdVerifComputed, kPwdVerifSize); diff --git a/CPP/7zip/Crypto/ZipCrypto.cpp b/CPP/7zip/Crypto/ZipCrypto.cpp index b6c987f5..6f767713 100644 --- a/CPP/7zip/Crypto/ZipCrypto.cpp +++ b/CPP/7zip/Crypto/ZipCrypto.cpp @@ -49,7 +49,7 @@ HRESULT CEncoder::WriteHeader_Check16(ISequentialOutStream *outStream, UInt16 cr PKZIP 2.0+ used 1 byte CRC check. It's more secure. We also use 1 byte CRC. */ - g_RandomGenerator.Generate(h, kHeaderSize - 1); + MY_RAND_GEN(h, kHeaderSize - 1); // h[kHeaderSize - 2] = (Byte)(crc); h[kHeaderSize - 1] = (Byte)(crc >> 8); diff --git a/CPP/7zip/Crypto/ZipStrong.cpp b/CPP/7zip/Crypto/ZipStrong.cpp index 152b469f..7bfb0741 100644 --- a/CPP/7zip/Crypto/ZipStrong.cpp +++ b/CPP/7zip/Crypto/ZipStrong.cpp @@ -85,13 +85,14 @@ HRESULT CDecoder::ReadHeader(ISequentialInStream *inStream, UInt32 crc, UInt64 u return E_NOTIMPL; RINOK(ReadStream_FALSE(inStream, temp, 4)); _remSize = GetUi32(temp); - const UInt32 kAlign = 16; + // const UInt32 kAlign = 16; if (_remSize < 16 || _remSize > (1 << 18)) return E_NOTIMPL; - if (_remSize + kAlign > _buf.Size()) + if (_remSize > _bufAligned.Size()) { - _buf.Alloc(_remSize + kAlign); - _bufAligned = (Byte *)((ptrdiff_t)((Byte *)_buf + kAlign - 1) & ~(ptrdiff_t)(kAlign - 1)); + _bufAligned.AllocAtLeast(_remSize); + if (!(Byte *)_bufAligned) + return E_OUTOFMEMORY; } return ReadStream_FALSE(inStream, _bufAligned, _remSize); } diff --git a/CPP/7zip/Crypto/ZipStrong.h b/CPP/7zip/Crypto/ZipStrong.h index b2a329de..39c18b1a 100644 --- a/CPP/7zip/Crypto/ZipStrong.h +++ b/CPP/7zip/Crypto/ZipStrong.h @@ -3,7 +3,7 @@ #ifndef __CRYPTO_ZIP_STRONG_H #define __CRYPTO_ZIP_STRONG_H -#include "../../Common/MyBuffer.h" +#include "../../Common/MyBuffer2.h" #include "../IPassword.h" @@ -35,8 +35,7 @@ class CBaseCoder: { protected: CKeyInfo _key; - CByteBuffer _buf; - Byte *_bufAligned; + CAlignedBuffer _bufAligned; public: STDMETHOD(Init)(); STDMETHOD(CryptoSetPassword)(const Byte *data, UInt32 size); diff --git a/CPP/7zip/ICoder.h b/CPP/7zip/ICoder.h index 54b24765..b2775ead 100644 --- a/CPP/7zip/ICoder.h +++ b/CPP/7zip/ICoder.h @@ -131,7 +131,23 @@ namespace NCoderPropID kBlockSize2, // VT_UI4 or VT_UI8 kCheckSize, // VT_UI4 : size of digest in bytes kFilter, // VT_BSTR - kMemUse // VT_UI8 + kMemUse, // VT_UI8 + + /* zstd props */ + kStrategy, // VT_UI4 1=ZSTD_fast, 2=ZSTD_dfast, 3=ZSTD_greedy, 4=ZSTD_lazy, 5=ZSTD_lazy2, 6=ZSTD_btlazy2, 7=ZSTD_btopt, 8=ZSTD_btultra + kFast, // VT_UI4 The minimum fast is 1 and the maximum is 64 (default: unused) + kLong, // VT_UI4 The minimum long is 10 (1KiB) and the maximum is 30 (1GiB) on x32 and 31 (2GiB) on x64 + kWindowLog, // VT_UI4 The minimum long is 10 (1KiB) and the maximum is 30 (1GiB) on x32 and 31 (2GiB) on x64 + kHashLog, // VT_UI4 The minimum hlog is 6 (64 B) and the maximum is 26 (128 MiB). + kChainLog, // VT_UI4 The minimum clog is 6 (64 B) and the maximum is 28 (256 MiB) + kSearchLog, // VT_UI4 The minimum slog is 1 and the maximum is 26 + kMinMatch, // VT_UI4 The minimum slen is 3 and the maximum is 7. + kTargetLen, // VT_UI4 The minimum tlen is 0 and the maximum is 999. + kOverlapLog, // VT_UI4 The minimum ovlog is 0 and the maximum is 9. (default: 6) + kLdmHashLog, // VT_UI4 The minimum ldmhlog is 6 and the maximum is 26 (default: 20). + kLdmSearchLength, // VT_UI4 The minimum ldmslen is 4 and the maximum is 4096 (default: 64). + kLdmBucketSizeLog, // VT_UI4 The minimum ldmblog is 0 and the maximum is 8 (default: 3). + kLdmHashRateLog // VT_UI4 The default value is wlog - ldmhlog. }; } diff --git a/CPP/7zip/IDecl.h b/CPP/7zip/IDecl.h index 41037dd8..2715d24d 100644 --- a/CPP/7zip/IDecl.h +++ b/CPP/7zip/IDecl.h @@ -6,15 +6,10 @@ #include "../Common/MyUnknown.h" #define k_7zip_GUID_Data1 0x23170F69 - -#ifdef NEED_7ZIP_GUID #define k_7zip_GUID_Data2 0x40C1 -#else -#define k_7zip_GUID_Data2 0x0803 -#endif +#define k_7zip_GUID_Data2_ZS 0x20BB #define k_7zip_GUID_Data3_Common 0x278A - #define k_7zip_GUID_Data3_Decoder 0x2790 #define k_7zip_GUID_Data3_Encoder 0x2791 #define k_7zip_GUID_Data3_Hasher 0x2792 diff --git a/CPP/7zip/LzmaDec.mak b/CPP/7zip/LzmaDec.mak index 3c0e7c5b..9aa30863 100644 --- a/CPP/7zip/LzmaDec.mak +++ b/CPP/7zip/LzmaDec.mak @@ -1,4 +1,4 @@ -!IF "$(CPU)" == "AMD64" +!IF "$(PLATFORM)" == "x64" CFLAGS_C_SPEC = -D_LZMA_DEC_OPT ASM_OBJS = $(ASM_OBJS) \ $O\LzmaDecOpt.obj diff --git a/CPP/7zip/UI/Client7z/Client7z.cpp b/CPP/7zip/UI/Client7z/Client7z.cpp index 775c8ef3..9a06cdc1 100644 --- a/CPP/7zip/UI/Client7z/Client7z.cpp +++ b/CPP/7zip/UI/Client7z/Client7z.cpp @@ -33,7 +33,7 @@ HINSTANCE g_hInstance = 0; // Tou can find the list of all GUIDs in Guid.txt file. // use another CLSIDs, if you want to support other formats (zip, rar, ...). -// {23170F69-0803-278A-1000-000110070000} +// {23170F69-40C1-278A-1000-000110070000} DEFINE_GUID(CLSID_CFormat7z, 0x23170F69, 0x40C1, 0x278A, 0x10, 0x00, 0x00, 0x01, 0x10, 0x07, 0x00, 0x00); diff --git a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp index d5096325..8d23ff21 100644 --- a/CPP/7zip/UI/Common/ArchiveCommandLine.cpp +++ b/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -23,6 +23,7 @@ #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" +#include "../../../Windows/ErrorMsg.h" #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" #ifdef _WIN32 @@ -39,7 +40,9 @@ extern bool g_CaseSensitive; extern bool g_PathTrailReplaceMode; +#ifdef _7ZIP_LARGE_PAGES bool g_LargePagesMode = false; +#endif #ifdef UNDER_CE @@ -410,8 +413,19 @@ static void AddToCensorFromListFile( UStringVector names; if (!NFind::DoesFileExist(us2fs(fileName))) throw CArcCmdLineException(kCannotFindListFile, fileName); - if (!ReadNamesFromListFile(us2fs(fileName), names, codePage)) + DWORD lastError = 0; + if (!ReadNamesFromListFile2(us2fs(fileName), names, codePage, lastError)) + { + if (lastError != 0) + { + UString m; + m = "The file operation error for listfile"; + m.Add_LF(); + m += NError::MyFormatMessage(lastError); + throw CArcCmdLineException(m, fileName); + } throw CArcCmdLineException(kIncorrectListFile, fileName); + } if (renamePairs) { if ((names.Size() & 1) != 0) diff --git a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp index 1119d1b4..aae08ada 100644 --- a/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveExtractCallback.cpp @@ -1182,7 +1182,9 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) bool needDelete = true; if (needDelete) { + if (NFind::DoesFileExist(fullProcessedPath)) if (!DeleteFileAlways(fullProcessedPath)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) { RINOK(SendMessageError_with_LastError(kCantDeleteOutputFile, fullProcessedPath)); return S_OK; @@ -1368,13 +1370,35 @@ if (askExtractMode == NArchive::NExtract::NAskMode::kExtract && !_testMode) // UInt64 ticks = GetCpuTicks(); bool res = _outFileStreamSpec->File.SetLength(_curSize); _fileLengthWasSet = res; - _outFileStreamSpec->File.SeekToBegin(); + // ticks = GetCpuTicks() - ticks; // printf("\nticks = %10d\n", (unsigned)ticks); if (!res) { RINOK(SendMessageError_with_LastError(kCantSetFileLen, fullProcessedPath)); } + + /* + _outFileStreamSpec->File.Close(); + ticks = GetCpuTicks() - ticks; + printf("\nticks = %10d\n", (unsigned)ticks); + return S_FALSE; + */ + + /* + File.SetLength() on FAT (xp64): is fast, but then File.Close() can be slow, + if we don't write any data. + File.SetLength() for remote share file (exFAT) can be slow in some cases, + and the Windows can return "network error" after 1 minute, + while remote file still can grow. + We need some way to detect such bad cases and disable PreAllocateOutFile mode. + */ + + res = _outFileStreamSpec->File.SeekToBegin(); + if (!res) + { + RINOK(SendMessageError_with_LastError("Can not seek to begin of file", fullProcessedPath)); + } } #ifdef SUPPORT_ALT_STREAMS diff --git a/CPP/7zip/UI/Common/ArchiveName.cpp b/CPP/7zip/UI/Common/ArchiveName.cpp index aa47f7af..b07d46c0 100644 --- a/CPP/7zip/UI/Common/ArchiveName.cpp +++ b/CPP/7zip/UI/Common/ArchiveName.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../../Common/Wildcard.h" + #include "../../../Windows/FileDir.h" #include "../../../Windows/FileName.h" @@ -11,7 +13,7 @@ using namespace NWindows; using namespace NFile; -UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) +static UString CreateArchiveName(const NFind::CFileInfo &fi, bool keepName) { FString resultName = fi.Name; if (!fi.IsDir() && !keepName) @@ -72,7 +74,82 @@ static FString CreateArchiveName2(const FString &path, bool fromPrev, bool keepN return resultName; } -UString CreateArchiveName(const UString &path, bool fromPrev, bool keepName) + +UString CreateArchiveName(const UStringVector &paths, const NFind::CFileInfo *fi) { - return Get_Correct_FsFile_Name(fs2us(CreateArchiveName2(us2fs(path), fromPrev, keepName))); + bool keepName = false; + /* + if (paths.Size() == 1) + { + const UString &name = paths[0]; + if (name.Len() > 4) + if (CompareFileNames(name.RightPtr(4), L".tar") == 0) + keepName = true; + } + */ + + UString name; + if (fi) + name = CreateArchiveName(*fi, keepName); + else + { + if (paths.IsEmpty()) + return L"archive"; + bool fromPrev = (paths.Size() > 1); + name = Get_Correct_FsFile_Name(fs2us(CreateArchiveName2(us2fs(paths.Front()), fromPrev, keepName))); + } + + UStringVector names; + + { + FOR_VECTOR (i, paths) + { + NFind::CFileInfo fi2; + const NFind::CFileInfo *fp; + if (fi && paths.Size() == 1) + fp = fi; + else + { + if (!fi2.Find(us2fs(paths[i]))) + continue; + fp = &fi2; + } + names.Add(fs2us(fp->Name)); + } + } + + UString postfix; + UInt32 index = 1; + + for (;;) + { + // we don't want cases when we include archive to itself. + // so we find first available name for archive + const UString name2 = name + postfix; + const UString name2_zip = name2 + L".zip"; + const UString name2_7z = name2 + L".7z"; + const UString name2_tar = name2 + L".tar"; + const UString name2_wim = name2 + L".wim"; + + unsigned i = 0; + + for (i = 0; i < names.Size(); i++) + { + const UString &fname = names[i]; + if ( 0 == CompareFileNames(fname, name2_zip) + || 0 == CompareFileNames(fname, name2_7z) + || 0 == CompareFileNames(fname, name2_tar) + || 0 == CompareFileNames(fname, name2_wim)) + break; + } + + if (i == names.Size()) + break; + index++; + postfix = "_"; + postfix.Add_UInt32(index); + } + + name += postfix; + return name; } diff --git a/CPP/7zip/UI/Common/ArchiveName.h b/CPP/7zip/UI/Common/ArchiveName.h index 7b49c7bb..ce2d1923 100644 --- a/CPP/7zip/UI/Common/ArchiveName.h +++ b/CPP/7zip/UI/Common/ArchiveName.h @@ -3,11 +3,8 @@ #ifndef __ARCHIVE_NAME_H #define __ARCHIVE_NAME_H -#include "../../../Common/MyString.h" - #include "../../../Windows/FileFind.h" -UString CreateArchiveName(const UString &path, bool fromPrev, bool keepName); -UString CreateArchiveName(const NWindows::NFile::NFind::CFileInfo &fileInfo, bool keepName); +UString CreateArchiveName(const UStringVector &paths, const NWindows::NFile::NFind::CFileInfo *fi = NULL); #endif diff --git a/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp b/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp index a9fda7b2..fd8f9f80 100644 --- a/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp +++ b/CPP/7zip/UI/Common/ArchiveOpenCallback.cpp @@ -102,7 +102,14 @@ STDMETHODIMP COpenCallbackImp::GetStream(const wchar_t *name, IInStream **inStre // if (!allowAbsVolPaths) if (!IsSafePath(name2)) return S_FALSE; - + + // #ifdef _WIN32 + // we don't want to support wildcards in names here here + if (name2.Find(L'?') >= 0 || + name2.Find(L'*') >= 0) + return S_FALSE; + // #endif + #endif diff --git a/CPP/7zip/UI/Common/Bench.cpp b/CPP/7zip/UI/Common/Bench.cpp index c0d0e549..3fb0758c 100644 --- a/CPP/7zip/UI/Common/Bench.cpp +++ b/CPP/7zip/UI/Common/Bench.cpp @@ -29,7 +29,6 @@ #endif #include "../../../../C/7zCrc.h" -#include "../../../../C/Alloc.h" #include "../../../../C/CpuArch.h" #ifndef _7ZIP_ST @@ -47,6 +46,7 @@ #include "../../../Common/IntToString.h" +#include "../../../Common/MyBuffer2.h" #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" @@ -94,80 +94,33 @@ static const UInt32 kAdditionalSize = (1 << 16); static const UInt32 kCompressedAdditionalSize = (1 << 10); static const UInt32 kMaxLzmaPropSize = 5; + + +#define ALLOC_WITH_HRESULT(_buffer_, _size_) \ + (_buffer_)->Alloc(_size_); \ + if (!(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; + + class CBaseRandomGenerator { UInt32 A1; UInt32 A2; + UInt32 Salt; public: - CBaseRandomGenerator() { Init(); } + CBaseRandomGenerator(UInt32 salt = 0): Salt(salt) { Init(); } void Init() { A1 = 362436069; A2 = 521288629;} UInt32 GetRnd() { - return + return Salt ^ + ( ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) + - ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ); + ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ) + ); } }; -static const unsigned kBufferAlignment = 1 << 4; - -struct CBenchBuffer -{ - size_t BufferSize; - - #ifdef _WIN32 - - Byte *Buffer; - - CBenchBuffer(): BufferSize(0), Buffer(NULL) {} - ~CBenchBuffer() { ::MidFree(Buffer); } - - void AllocAlignedMask(size_t size, size_t) - { - ::MidFree(Buffer); - BufferSize = 0; - Buffer = (Byte *)::MidAlloc(size); - if (Buffer) - BufferSize = size; - } - - #else - - Byte *Buffer; - Byte *_bufBase; - - CBenchBuffer(): BufferSize(0), Buffer(NULL), _bufBase(NULL){} - ~CBenchBuffer() { ::MidFree(_bufBase); } - - void AllocAlignedMask(size_t size, size_t alignMask) - { - ::MidFree(_bufBase); - Buffer = NULL; - BufferSize = 0; - _bufBase = (Byte *)::MidAlloc(size + alignMask); - - if (_bufBase) - { - // Buffer = (Byte *)(((uintptr_t)_bufBase + alignMask) & ~(uintptr_t)alignMask); - Buffer = (Byte *)(((ptrdiff_t)_bufBase + alignMask) & ~(ptrdiff_t)alignMask); - BufferSize = size; - } - } - - #endif - - bool Alloc(size_t size) - { - if (Buffer && BufferSize == size) - return true; - AllocAlignedMask(size, kBufferAlignment - 1); - return (Buffer != NULL || size == 0); - } -}; - - -class CBenchRandomGenerator: public CBenchBuffer +class CBenchRandomGenerator: public CAlignedBuffer { static UInt32 GetVal(UInt32 &res, unsigned numBits) { @@ -184,23 +137,22 @@ class CBenchRandomGenerator: public CBenchBuffer public: - void GenerateSimpleRandom(CBaseRandomGenerator *_RG_) + void GenerateSimpleRandom(UInt32 salt) { - CBaseRandomGenerator rg = *_RG_; - const size_t bufSize = BufferSize; - Byte *buf = Buffer; + CBaseRandomGenerator rg(salt); + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; for (size_t i = 0; i < bufSize; i++) buf[i] = (Byte)rg.GetRnd(); - *_RG_ = rg; } - void GenerateLz(unsigned dictBits, CBaseRandomGenerator *_RG_) + void GenerateLz(unsigned dictBits, UInt32 salt) { - CBaseRandomGenerator rg = *_RG_; + CBaseRandomGenerator rg(salt); UInt32 pos = 0; UInt32 rep0 = 1; - const size_t bufSize = BufferSize; - Byte *buf = Buffer; + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; unsigned posBits = 1; while (pos < bufSize) @@ -255,8 +207,6 @@ public: *dest++ = *src++; } } - - *_RG_ = rg; } }; @@ -297,7 +247,7 @@ STDMETHODIMP CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processed class CBenchmarkOutStream: public ISequentialOutStream, - public CBenchBuffer, + public CAlignedBuffer, public CMyUnknownImp { // bool _overflow; @@ -325,13 +275,13 @@ public: STDMETHODIMP CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize) { - size_t curSize = BufferSize - Pos; + size_t curSize = Size() - Pos; if (curSize > size) curSize = size; if (curSize != 0) { if (RealCopy) - memcpy(Buffer + Pos, data, curSize); + memcpy(((Byte *)*this) + Pos, data, curSize); if (CalcCrc) Crc = CrcUpdate(Crc, data, curSize); Pos += curSize; @@ -522,10 +472,9 @@ class CBenchProgressInfo: { public: CBenchProgressStatus *Status; - HRESULT Res; IBenchCallback *Callback; - CBenchProgressInfo(): Callback(0) {} + CBenchProgressInfo(): Callback(NULL) {} MY_UNKNOWN_IMP STDMETHOD(SetRatioInfo)(const UInt64 *inSize, const UInt64 *outSize); }; @@ -687,20 +636,39 @@ UInt64 GetDecompressRating(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt return props.GetDecompressRating(elapsedTime, freq, outSize, inSize, numIterations); } + + + +#ifndef _7ZIP_ST +struct CBenchSyncCommon +{ + bool ExitMode; + NSynchronization::CManualResetEvent StartEvent; + + CBenchSyncCommon(): ExitMode(false) {} +}; +#endif + + struct CEncoderInfo; struct CEncoderInfo { #ifndef _7ZIP_ST NWindows::CThread thread[2]; + NSynchronization::CManualResetEvent ReadyEvent; UInt32 NumDecoderSubThreads; + CBenchSyncCommon *Common; #endif + CMyComPtr _encoder; CMyComPtr _encoderFilter; CBenchProgressInfo *progressInfoSpec[2]; CMyComPtr progressInfo[2]; UInt64 NumIterations; + UInt32 Salt; + #ifdef USE_ALLOCA size_t AllocaSize; #endif @@ -739,26 +707,29 @@ struct CEncoderInfo const Byte *fileData; CBenchRandomGenerator rg; - CBenchBuffer rgCopy; // it must be 16-byte aligned !!! + CAlignedBuffer rgCopy; // it must be 16-byte aligned !!! CBenchmarkOutStream *propStreamSpec; CMyComPtr propStream; - // for decode + unsigned generateDictBits; COneMethodInfo _method; + + // for decode size_t _uncompressedDataSize; - HRESULT Init( - const COneMethodInfo &method, - unsigned generateDictBits, - CBaseRandomGenerator *rg); + HRESULT Generate(); HRESULT Encode(); HRESULT Decode(UInt32 decoderIndex); CEncoderInfo(): + #ifndef _7ZIP_ST + Common(NULL), + #endif + Salt(0), fileData(NULL), CheckCrc_Enc(true), CheckCrc_Dec(true), - outStreamSpec(0), callback(0), printCallback(0), propStreamSpec(0) {} + outStreamSpec(NULL), callback(NULL), printCallback(NULL), propStreamSpec(NULL) {} #ifndef _7ZIP_ST @@ -773,14 +744,15 @@ struct CEncoderInfo #endif res = encoder->Encode(); - encoder->Results[0] = res; } catch(...) { res = E_FAIL; } + encoder->Results[0] = res; if (res != S_OK) encoder->progressInfoSpec[0]->Status->SetResult(res); + encoder->ReadyEvent.Set(); return 0; } @@ -799,7 +771,12 @@ struct CEncoderInfo HRESULT CreateEncoderThread() { - return thread[0].Create(EncodeThreadFunction, this); + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = thread[0].Create(EncodeThreadFunction, this); + return HRESULT_FROM_WIN32(res); } HRESULT CreateDecoderThread(unsigned index, bool callbackMode @@ -824,11 +801,10 @@ struct CEncoderInfo }; -HRESULT CEncoderInfo::Init( - const COneMethodInfo &method, - unsigned generateDictBits, - CBaseRandomGenerator *rgLoc) +HRESULT CEncoderInfo::Generate() { + const COneMethodInfo &method = _method; + // we need extra space, if input data is already compressed const size_t kCompressedBufferSize = kCompressedAdditionalSize + @@ -842,40 +818,39 @@ HRESULT CEncoderInfo::Init( if (!fileData) { - if (!rg.Alloc(kBufferSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&rg, kBufferSize); // DWORD ttt = GetTickCount(); if (generateDictBits == 0) - rg.GenerateSimpleRandom(rgLoc); + rg.GenerateSimpleRandom(Salt); else - rg.GenerateLz(generateDictBits, rgLoc); + rg.GenerateLz(generateDictBits, Salt); // printf("\n%d\n ", GetTickCount() - ttt); - crc = CrcCalc(rg.Buffer, rg.BufferSize); - uncompressedDataPtr = rg.Buffer; + crc = CrcCalc((const Byte *)rg, rg.Size()); + uncompressedDataPtr = (const Byte *)rg; } if (_encoderFilter) { - if (!rgCopy.Alloc(kBufferSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&rgCopy, kBufferSize); } - outStreamSpec = new CBenchmarkOutStream; - outStream = outStreamSpec; - if (!outStreamSpec->Alloc(kCompressedBufferSize)) - return E_OUTOFMEMORY; + if (!outStream) + { + outStreamSpec = new CBenchmarkOutStream; + outStream = outStreamSpec; + } + + ALLOC_WITH_HRESULT(outStreamSpec, kCompressedBufferSize) - propStreamSpec = 0; if (!propStream) { propStreamSpec = new CBenchmarkOutStream; propStream = propStreamSpec; } - if (!propStreamSpec->Alloc(kMaxLzmaPropSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(propStreamSpec, kMaxLzmaPropSize); propStreamSpec->Init(true, false); @@ -962,6 +937,28 @@ static void My_FilterBench(ICompressFilter *filter, Byte *data, size_t size) HRESULT CEncoderInfo::Encode() { + RINOK(Generate()); + + #ifndef _7ZIP_ST + if (Common) + { + Results[0] = S_OK; + WRes wres = ReadyEvent.Set(); + if (wres == 0) + wres = Common->StartEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + if (Common->ExitMode) + return S_OK; + } + else + #endif + { + CBenchProgressInfo *bpi = progressInfoSpec[0]; + bpi->SetStartTime(); + } + + CBenchInfo &bi = progressInfoSpec[0]->BenchInfo; bi.UnpackSize = 0; bi.PackSize = 0; @@ -998,10 +995,10 @@ HRESULT CEncoderInfo::Encode() if (_encoderFilter) { - memcpy(rgCopy.Buffer, uncompressedDataPtr, kBufferSize); + memcpy((Byte *)rgCopy, uncompressedDataPtr, kBufferSize); _encoderFilter->Init(); - My_FilterBench(_encoderFilter, rgCopy.Buffer, kBufferSize); - RINOK(WriteStream(outStream, rgCopy.Buffer, kBufferSize)); + My_FilterBench(_encoderFilter, (Byte *)rgCopy, kBufferSize); + RINOK(WriteStream(outStream, (const Byte *)rgCopy, kBufferSize)); } else { @@ -1079,7 +1076,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (setDecProps) { - RINOK(setDecProps->SetDecoderProperties2(propStreamSpec->Buffer, (UInt32)propStreamSpec->Pos)); + RINOK(setDecProps->SetDecoderProperties2((const Byte *)*propStreamSpec, (UInt32)propStreamSpec->Pos)); } { @@ -1107,7 +1104,7 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) prev = pi->BenchInfo.UnpackSize; } - inStreamSpec->Init(outStreamSpec->Buffer, compressedSize); + inStreamSpec->Init((const Byte *)*outStreamSpec, compressedSize); crcOutStreamSpec->Init(); UInt64 outSize = kBufferSize; @@ -1115,12 +1112,12 @@ HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) if (_decoderFilter) { - if (compressedSize > rgCopy.BufferSize) + if (compressedSize > rgCopy.Size()) return E_FAIL; - memcpy(rgCopy.Buffer, outStreamSpec->Buffer, compressedSize); + memcpy((Byte *)rgCopy, (const Byte *)*outStreamSpec, compressedSize); _decoderFilter->Init(); - My_FilterBench(_decoderFilter, rgCopy.Buffer, compressedSize); - RINOK(WriteStream(crcOutStream, rgCopy.Buffer, compressedSize)); + My_FilterBench(_decoderFilter, (Byte *)rgCopy, compressedSize); + RINOK(WriteStream(crcOutStream, (const Byte *)rgCopy, compressedSize)); } else { @@ -1144,7 +1141,7 @@ static const UInt32 kNumThreadsMax = (1 << 12); struct CBenchEncoders { CEncoderInfo *encoders; - CBenchEncoders(UInt32 num): encoders(0) { encoders = new CEncoderInfo[num]; } + CBenchEncoders(UInt32 num): encoders(NULL) { encoders = new CEncoderInfo[num]; } ~CBenchEncoders() { delete []encoders; } }; @@ -1158,6 +1155,57 @@ static UInt64 GetNumIterations(UInt64 numCommands, UInt64 complexInCommands) } + +#ifndef _7ZIP_ST + +// ---------- CBenchThreadsFlusher ---------- + +struct CBenchThreadsFlusher +{ + CBenchEncoders *EncodersSpec; + CBenchSyncCommon Common; + unsigned NumThreads; + bool NeedClose; + + CBenchThreadsFlusher(): NumThreads(0), NeedClose(false) {} + + ~CBenchThreadsFlusher() + { + StartAndWait(true); + } + + WRes StartAndWait(bool exitMode = false); +}; + + +WRes CBenchThreadsFlusher::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes res = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + NWindows::CThread &t = EncodersSpec->encoders[i].thread[0]; + if (t.IsCreated()) + { + WRes res2 = t.Wait(); + if (res2 == 0) + res2 = t.Close(); + if (res == S_OK) + res = res2; + } + } + NeedClose = false; + return res; +} + +#endif + + + static HRESULT MethodBench( DECL_EXTERNAL_CODECS_LOC_VARS UInt64 complexInCommands, @@ -1209,6 +1257,8 @@ static HRESULT MethodBench( numSubDecoderThreads = 2; } } + + bool mtEncMode = (numEncoderThreads > 1); #endif CBenchEncoders encodersSpec(numEncoderThreads); @@ -1248,9 +1298,6 @@ static HRESULT MethodBench( } } - CBaseRandomGenerator rg; - rg.Init(); - UInt32 crc = 0; if (fileData) crc = CrcCalc(fileData, uncompressedDataSize); @@ -1259,22 +1306,38 @@ static HRESULT MethodBench( { CEncoderInfo &encoder = encoders[i]; encoder._method = method; + encoder.generateDictBits = generateDictBits; encoder._uncompressedDataSize = uncompressedDataSize; encoder.kBufferSize = uncompressedDataSize; encoder.fileData = fileData; encoder.crc = crc; - - RINOK(encoders[i].Init(method, generateDictBits, &rg)); } CBenchProgressStatus status; status.Res = S_OK; status.EncodeMode = true; + #ifndef _7ZIP_ST + CBenchThreadsFlusher encoderFlusher; + if (mtEncMode) + { + WRes wres = encoderFlusher.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + encoderFlusher.NumThreads = numEncoderThreads; + encoderFlusher.EncodersSpec = &encodersSpec; + encoderFlusher.NeedClose = true; + } + #endif + for (i = 0; i < numEncoderThreads; i++) { CEncoderInfo &encoder = encoders[i]; encoder.NumIterations = GetNumIterations(benchProps->GeComprCommands(uncompressedDataSize), complexInCommands); + encoder.Salt = g_CrcTable[i & 0xFF]; + encoder.Salt ^= (g_CrcTable[(i >> 8) & 0xFF] << 3); + // (g_CrcTable[0] == 0), and (encoder.Salt == 0) for first thread + // printf(" %8x", encoder.Salt); for (int j = 0; j < 2; j++) { @@ -1289,30 +1352,50 @@ static HRESULT MethodBench( CBenchProgressInfo *bpi = encoder.progressInfoSpec[0]; bpi->Callback = callback; bpi->BenchInfo.NumIterations = numEncoderThreads; - bpi->SetStartTime(); } #ifndef _7ZIP_ST - if (numEncoderThreads > 1) + if (mtEncMode) { #ifdef USE_ALLOCA encoder.AllocaSize = (i * 16 * 21) & 0x7FF; #endif + encoder.Common = &encoderFlusher.Common; RINOK(encoder.CreateEncoderThread()) } - else #endif - { - RINOK(encoder.Encode()); - } } - + + if (printCallback) + { + RINOK(printCallback->CheckBreak()); + } + #ifndef _7ZIP_ST - if (numEncoderThreads > 1) + if (mtEncMode) + { for (i = 0; i < numEncoderThreads; i++) - encoders[i].thread[0].Wait(); + { + CEncoderInfo &encoder = encoders[i]; + WRes wres = encoder.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(encoder.Results[0]); + } + + CBenchProgressInfo *bpi = encoders[0].progressInfoSpec[0]; + bpi->SetStartTime(); + + WRes wres = encoderFlusher.StartAndWait(); + if (status.Res == 0 && wres != 0) + return HRESULT_FROM_WIN32(wres); + } + else #endif + { + RINOK(encoders[0].Encode()); + } RINOK(status.Res); @@ -1328,11 +1411,16 @@ static HRESULT MethodBench( CEncoderInfo &encoder = encoders[i]; info.UnpackSize += encoder.kBufferSize; info.PackSize += encoder.compressedSize; + // printf("\n%7d\n", encoder.compressedSize); } RINOK(callback->SetEncodeResult(info, true)); + + + // ---------- Decode ---------- + status.Res = S_OK; status.EncodeMode = false; @@ -1545,7 +1633,7 @@ struct CFreqThreads CFreqInfo *Items; UInt32 NumThreads; - CFreqThreads(): Items(0), NumThreads(0) {} + CFreqThreads(): Items(NULL), NumThreads(0) {} void WaitAll() { for (UInt32 i = 0; i < NumThreads; i++) @@ -1603,7 +1691,7 @@ struct CCrcThreads CCrcInfo *Items; UInt32 NumThreads; - CCrcThreads(): Items(0), NumThreads(0) {} + CCrcThreads(): Items(NULL), NumThreads(0) {} void WaitAll() { for (UInt32 i = 0; i < NumThreads; i++) @@ -1619,21 +1707,21 @@ struct CCrcThreads #endif -static UInt32 CrcCalc1(const Byte *buf, UInt32 size) +static UInt32 CrcCalc1(const Byte *buf, size_t size) { UInt32 crc = CRC_INIT_VAL;; - for (UInt32 i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) crc = CRC_UPDATE_BYTE(crc, buf[i]); return CRC_GET_DIGEST(crc); } -static void RandGen(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) +static void RandGen(Byte *buf, size_t size, CBaseRandomGenerator &RG) { - for (UInt32 i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) buf[i] = (Byte)RG.GetRnd(); } -static UInt32 RandGenCrc(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) +static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) { RandGen(buf, size, RG); return CrcCalc1(buf, size); @@ -1641,14 +1729,15 @@ static UInt32 RandGenCrc(Byte *buf, UInt32 size, CBaseRandomGenerator &RG) bool CrcInternalTest() { - CBenchBuffer buffer; - const UInt32 kBufferSize0 = (1 << 8); - const UInt32 kBufferSize1 = (1 << 10); - const UInt32 kCheckSize = (1 << 5); - if (!buffer.Alloc(kBufferSize0 + kBufferSize1)) + CAlignedBuffer buffer; + const size_t kBufferSize0 = (1 << 8); + const size_t kBufferSize1 = (1 << 10); + const unsigned kCheckSize = (1 << 5); + buffer.Alloc(kBufferSize0 + kBufferSize1); + if (!buffer.IsAllocated()) return false; - Byte *buf = buffer.Buffer; - UInt32 i; + Byte *buf = (Byte *)buffer; + size_t i; for (i = 0; i < kBufferSize0; i++) buf[i] = (Byte)i; UInt32 crc1 = CrcCalc1(buf, kBufferSize0); @@ -1657,7 +1746,7 @@ bool CrcInternalTest() CBaseRandomGenerator RG; RandGen(buf + kBufferSize0, kBufferSize1, RG); for (i = 0; i < kBufferSize0 + kBufferSize1 - kCheckSize; i++) - for (UInt32 j = 0; j < kCheckSize; j++) + for (unsigned j = 0; j < kCheckSize; j++) if (CrcCalc1(buf + i, j) != CrcCalc(buf + i, j)) return false; return true; @@ -1885,8 +1974,55 @@ AString GetProcessThreadsInfo(const NSystem::CProcessAffinity &ti) } +static void PrintSize(AString &s, UInt64 v) +{ + char c = 0; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; + }}}} + else + { + PrintHex(s, v); + return; + } + char temp[32]; + ConvertUInt64ToString(v, temp); + s += temp; + if (c) + s += c; +} + + +#ifdef _7ZIP_LARGE_PAGES + extern bool g_LargePagesMode; +extern "C" +{ + extern SIZE_T g_LargePageSize; +} + +void Add_LargePages_String(AString &s) +{ + if (g_LargePagesMode || g_LargePageSize != 0) + { + s += " (LP-"; + PrintSize(s, g_LargePageSize); + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_PageGB()) + s += "-1G"; + #endif + if (!g_LargePagesMode) + s += "-NA"; + s += ")"; + } +} + +#endif + + static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, bool size_Defined, UInt64 size, const char *threadsString, UInt32 numThreads) @@ -1898,8 +2034,15 @@ static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, else f.Print(" ?"); f.Print(" MB"); - if (g_LargePagesMode) - f.Print(" LP"); + + #ifdef _7ZIP_LARGE_PAGES + { + AString s; + Add_LargePages_String(s); + f.Print(s); + } + #endif + f.Print(", # "); f.Print(threadsString); PrintNumber(f, numThreads, 3); @@ -2189,14 +2332,13 @@ static HRESULT CrcBench( methodName, hashID)) return E_NOTIMPL; - CBenchBuffer buffer; + CAlignedBuffer buffer; size_t totalSize = (size_t)bufferSize * numThreads; if (totalSize / numThreads != bufferSize) return E_OUTOFMEMORY; - if (!buffer.Alloc(totalSize)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&buffer, totalSize) - Byte *buf = buffer.Buffer; + Byte *buf = (Byte *)buffer; CBaseRandomGenerator RG; UInt32 bsize = (bufferSize == 0 ? 1 : bufferSize); UInt64 numIterations = complexInCommands * 256 / complexity / bsize; @@ -2539,26 +2681,7 @@ static const char * const k_PF[] = #endif -static void PrintSize(AString &s, UInt64 v) -{ - char c = 0; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; - if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; - }}}} - else - { - PrintHex(s, v); - return; - } - char temp[32]; - ConvertUInt64ToString(v, temp); - s += temp; - if (c) - s += c; -} - + static void PrintPage(AString &s, UInt32 v) { @@ -2707,8 +2830,9 @@ void GetCpuName(AString &s) #endif - if (g_LargePagesMode) - s += " (LP)"; + #ifdef _7ZIP_LARGE_PAGES + Add_LargePages_String(s); + #endif } @@ -2762,7 +2886,7 @@ HRESULT Bench( bool multiDict) { if (!CrcInternalTest()) - return S_FALSE; + return E_FAIL; UInt32 numCPUs = 1; UInt64 ramSize = (UInt64)(sizeof(size_t)) << 29; @@ -2791,7 +2915,7 @@ HRESULT Bench( COneMethodInfo method; - CBenchBuffer fileDataBuffer; + CAlignedBuffer fileDataBuffer; { unsigned i; @@ -2816,10 +2940,9 @@ HRESULT Bench( return E_FAIL; if (len >= ((UInt32)1 << 31) || len == 0) return E_INVALIDARG; - if (!fileDataBuffer.Alloc((size_t)len)) - return E_OUTOFMEMORY; + ALLOC_WITH_HRESULT(&fileDataBuffer, (size_t)len); UInt32 processedSize; - file.Read(fileDataBuffer.Buffer, (UInt32)len, processedSize); + file.Read((Byte *)fileDataBuffer, (UInt32)len, processedSize); if (processedSize != len) return E_FAIL; if (printCallback) @@ -2968,6 +3091,9 @@ HRESULT Bench( UInt64 start = ::GetTimeCount(); UInt32 sum = (UInt32)start; sum = CountCpuFreq(sum, (UInt32)(numMilCommands * 1000000 / kNumFreqCommands), g_BenchCpuFreqTemp); + if (sum == 0xF1541213) + if (printCallback) + printCallback->Print(""); const UInt64 realDelta = ::GetTimeCount() - start; start = realDelta; if (start == 0) @@ -2984,7 +3110,7 @@ HRESULT Bench( else { // PrintNumber(*printCallback, start, 0); - PrintNumber(*printCallback, mipsVal, 5 + ((sum == 0xF1541213) ? 1 : 0)); + PrintNumber(*printCallback, mipsVal, 5); } } /* @@ -3032,7 +3158,7 @@ HRESULT Bench( complexInCommands, true, numThreadsSpecified, method, - uncompressedDataSize, fileDataBuffer.Buffer, + uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, benchCallback, &benchProps); } @@ -3344,9 +3470,9 @@ HRESULT Bench( { res = TotalBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, numThreads, - dictIsDefined || fileDataBuffer.Buffer, // forceUnpackSize - fileDataBuffer.Buffer ? fileDataBuffer.BufferSize : dict, - fileDataBuffer.Buffer, + dictIsDefined || fileDataBuffer.IsAllocated(), // forceUnpackSize + fileDataBuffer.IsAllocated() ? fileDataBuffer.Size() : dict, + (const Byte *)fileDataBuffer, printCallback, &callback); RINOK(res); } @@ -3436,9 +3562,9 @@ HRESULT Bench( } size_t uncompressedDataSize; - if (fileDataBuffer.Buffer) + if (fileDataBuffer.IsAllocated()) { - uncompressedDataSize = fileDataBuffer.BufferSize; + uncompressedDataSize = fileDataBuffer.Size(); } else { @@ -3452,7 +3578,7 @@ HRESULT Bench( complexInCommands, true, numThreads, method2, - uncompressedDataSize, fileDataBuffer.Buffer, + uncompressedDataSize, (const Byte *)fileDataBuffer, kOldLzmaDictBits, printCallback, &callback, &callback.BenchProps); f.NewLine(); RINOK(res); diff --git a/CPP/7zip/UI/Common/Bench.h b/CPP/7zip/UI/Common/Bench.h index 1990aab0..1d052aac 100644 --- a/CPP/7zip/UI/Common/Bench.h +++ b/CPP/7zip/UI/Common/Bench.h @@ -68,5 +68,10 @@ void GetSysInfo(AString &s1, AString &s2); void GetCpuName(AString &s); void GetCpuFeatures(AString &s); +#ifdef _7ZIP_LARGE_PAGES +void Add_LargePages_String(AString &s); +#else +// #define Add_LargePages_String +#endif #endif diff --git a/CPP/7zip/UI/Common/EnumDirItems.cpp b/CPP/7zip/UI/Common/EnumDirItems.cpp index 032e2fff..2c941e12 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.cpp +++ b/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -1084,3 +1084,13 @@ CMessagePathException::CMessagePathException(const char *a, const wchar_t *u) (*this) += u; } } + +CMessagePathException::CMessagePathException(const wchar_t *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} diff --git a/CPP/7zip/UI/Common/EnumDirItems.h b/CPP/7zip/UI/Common/EnumDirItems.h index 62205005..ae1d2269 100644 --- a/CPP/7zip/UI/Common/EnumDirItems.h +++ b/CPP/7zip/UI/Common/EnumDirItems.h @@ -22,6 +22,7 @@ HRESULT EnumerateItems( struct CMessagePathException: public UString { CMessagePathException(const char *a, const wchar_t *u = NULL); + CMessagePathException(const wchar_t *a, const wchar_t *u = NULL); }; diff --git a/CPP/7zip/UI/Common/HashCalc.cpp b/CPP/7zip/UI/Common/HashCalc.cpp index 9c0a1d0a..822018d6 100644 --- a/CPP/7zip/UI/Common/HashCalc.cpp +++ b/CPP/7zip/UI/Common/HashCalc.cpp @@ -230,7 +230,7 @@ HRESULT HashCalc( unsigned i; CHashBundle hb; RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS options.Methods)); - hb.Init(); + // hb.Init(); hb.NumErrors = dirItems.Stat.NumErrors; diff --git a/CPP/7zip/UI/Common/HashCalc.h b/CPP/7zip/UI/Common/HashCalc.h index 77373b85..524bd3bd 100644 --- a/CPP/7zip/UI/Common/HashCalc.h +++ b/CPP/7zip/UI/Common/HashCalc.h @@ -51,9 +51,13 @@ struct CHashBundle: public IHashCalc UInt64 CurSize; + UString MainName; + UString FirstFileName; + HRESULT SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVector &methods); - void Init() + // void Init() {} + CHashBundle() { NumDirs = NumFiles = NumAltStreams = FilesSize = AltStreamsSize = NumErrors = 0; } @@ -76,7 +80,7 @@ struct CHashBundle: public IHashCalc virtual HRESULT GetStream(const wchar_t *name, bool isFolder) x; \ virtual HRESULT OpenFileError(const FString &path, DWORD systemError) x; \ virtual HRESULT SetOperationResult(UInt64 fileSize, const CHashBundle &hb, bool showHash) x; \ - virtual HRESULT AfterLastFile(const CHashBundle &hb) x; \ + virtual HRESULT AfterLastFile(CHashBundle &hb) x; \ struct IHashCallbackUI: public IDirItemsCallback { diff --git a/CPP/7zip/UI/Common/OpenArchive.cpp b/CPP/7zip/UI/Common/OpenArchive.cpp index dcb81a64..e35c0e7d 100644 --- a/CPP/7zip/UI/Common/OpenArchive.cpp +++ b/CPP/7zip/UI/Common/OpenArchive.cpp @@ -563,6 +563,8 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa UInt32 parentType = 0; RINOK(GetRawProps->GetParent(curIndex, &curParent, &parentType)); + // 18.06: fixed : we don't want to split name to parts + /* if (parentType != NParentType::kAltStream) { for (;;) @@ -576,6 +578,7 @@ HRESULT CArc::GetItemPathToParent(UInt32 index, UInt32 parent, UStringVector &pa s.DeleteFrom(pos); } } + */ parts.Insert(0, s); @@ -2014,7 +2017,7 @@ HRESULT CArc::OpenStream2(const COpenOptions &op) } else { - const CArcInfoEx &ai = op.codecs->Formats[formatIndex]; + const CArcInfoEx &ai = op.codecs->Formats[(unsigned)formatIndex]; if (ai.FindExtension(extension) >= 0) { if (ai.Flags_FindSignature() && searchMarkerInHandler) diff --git a/CPP/7zip/UI/Common/Update.cpp b/CPP/7zip/UI/Common/Update.cpp index fc1eede4..2f1b3656 100644 --- a/CPP/7zip/UI/Common/Update.cpp +++ b/CPP/7zip/UI/Common/Update.cpp @@ -288,29 +288,27 @@ void CArchivePath::ParseFromPath(const UString &path, EArcNameMode mode) if (mode == k_ArcNameMode_Add) return; - if (mode == k_ArcNameMode_Exact) - { - BaseExtension.Empty(); - return; - } - int dotPos = Name.ReverseFind_Dot(); - if (dotPos < 0) - return; - if ((unsigned)dotPos == Name.Len() - 1) + if (mode != k_ArcNameMode_Exact) { - Name.DeleteBack(); - BaseExtension.Empty(); - return; + int dotPos = Name.ReverseFind_Dot(); + if (dotPos < 0) + return; + if ((unsigned)dotPos == Name.Len() - 1) + Name.DeleteBack(); + else + { + const UString ext = Name.Ptr(dotPos + 1); + if (BaseExtension.IsEqualTo_NoCase(ext)) + { + BaseExtension = ext; + Name.DeleteFrom(dotPos); + return; + } + } } - const UString ext = Name.Ptr(dotPos + 1); - if (BaseExtension.IsEqualTo_NoCase(ext)) - { - BaseExtension = ext; - Name.DeleteFrom(dotPos); - } - else - BaseExtension.Empty(); + + BaseExtension.Empty(); } UString CArchivePath::GetFinalPath() const @@ -327,6 +325,7 @@ UString CArchivePath::GetFinalPath() const UString CArchivePath::GetFinalVolPath() const { UString path = GetPathWithoutExt(); + // if BaseExtension is empty, we must ignore VolExtension also. if (!BaseExtension.IsEmpty()) { path += '.'; @@ -1166,7 +1165,7 @@ HRESULT UpdateArchive( { errorInfo.SystemError = ERROR_ACCESS_DENIED; errorInfo.Message = "The file is read-only"; - errorInfo.FileNames.Add(arcPath); + errorInfo.FileNames.Add(us2fs(arcPath)); return errorInfo.Get_HRESULT_Error(); } @@ -1377,6 +1376,31 @@ HRESULT UpdateArchive( unsigned ci; + + // self including protection + if (options.DeleteAfterCompressing) + { + for (ci = 0; ci < options.Commands.Size(); ci++) + { + CArchivePath &ap = options.Commands[ci].ArchivePath; + const FString path = us2fs(ap.GetFinalPath()); + // maybe we must compare absolute paths path here + FOR_VECTOR (i, dirItems.Items) + { + const FString phyPath = dirItems.GetPhyPath(i); + if (phyPath == path) + { + UString s; + s = "It is not allowed to include archive to itself"; + s.Add_LF(); + s += path; + throw s; + } + } + } + } + + for (ci = 0; ci < options.Commands.Size(); ci++) { CArchivePath &ap = options.Commands[ci].ArchivePath; @@ -1562,26 +1586,39 @@ HRESULT UpdateArchive( } CCurrentDirRestorer curDirRestorer; + + AStringVector paths; + AStringVector names; for (i = 0; i < fullPaths.Size(); i++) { const UString arcPath2 = fs2us(fullPaths[i]); const UString fileName = ExtractFileNameFromPath(arcPath2); - const AString path (GetAnsiString(arcPath2)); - const AString name (GetAnsiString(fileName)); + paths.Add(GetAnsiString(arcPath2)); + names.Add(GetAnsiString(fileName)); + // const AString path (GetAnsiString(arcPath2)); + // const AString name (GetAnsiString(fileName)); // Warning!!! MAPISendDocuments function changes Current directory // fnSend(0, ";", (LPSTR)(LPCSTR)path, (LPSTR)(LPCSTR)name, 0); + } - MapiFileDesc f; + CRecordVector files; + files.ClearAndSetSize(paths.Size()); + + for (i = 0; i < paths.Size(); i++) + { + MapiFileDesc &f = files[i]; memset(&f, 0, sizeof(f)); f.nPosition = 0xFFFFFFFF; - f.lpszPathName = (char *)(const char *)path; - f.lpszFileName = (char *)(const char *)name; - + f.lpszPathName = (char *)(const char *)paths[i]; + f.lpszFileName = (char *)(const char *)names[i]; + } + + { MapiMessage m; memset(&m, 0, sizeof(m)); - m.nFileCount = 1; - m.lpFiles = &f; + m.nFileCount = files.Size(); + m.lpFiles = &files.Front(); const AString addr (GetAnsiString(options.EMailAddress)); MapiRecipDesc rec; diff --git a/CPP/7zip/UI/Console/Console.dsp b/CPP/7zip/UI/Console/Console.dsp index 09ed568f..3da9da50 100644 --- a/CPP/7zip/UI/Console/Console.dsp +++ b/CPP/7zip/UI/Console/Console.dsp @@ -417,6 +417,10 @@ SOURCE=..\..\..\Common\MyBuffer.h # End Source File # Begin Source File +SOURCE=..\..\..\Common\MyBuffer2.h +# End Source File +# Begin Source File + SOURCE=..\..\..\Common\MyCom.h # End Source File # Begin Source File @@ -843,6 +847,15 @@ SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/UI/Console/Console.mak b/CPP/7zip/UI/Console/Console.mak index 6757e6b4..d4268c54 100644 --- a/CPP/7zip/UI/Console/Console.mak +++ b/CPP/7zip/UI/Console/Console.mak @@ -1,3 +1,9 @@ +MY_CONSOLE = 1 + +!IFNDEF UNDER_CE +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE +!ENDIF + CONSOLE_OBJS = \ $O\BenchCon.obj \ $O\ConsoleClose.obj \ @@ -33,4 +39,5 @@ UI_COMMON_OBJS = \ $O\UpdatePair.obj \ $O\UpdateProduce.obj \ -# +C_OBJS = $(C_OBJS) \ + $O\DllSecur.obj \ diff --git a/CPP/7zip/UI/Console/HashCon.cpp b/CPP/7zip/UI/Console/HashCon.cpp index ec8e6dca..3ade0fda 100644 --- a/CPP/7zip/UI/Console/HashCon.cpp +++ b/CPP/7zip/UI/Console/HashCon.cpp @@ -332,7 +332,7 @@ void CHashCallbackConsole::PrintProperty(const char *name, UInt64 value) *_so << name << s << endl; } -HRESULT CHashCallbackConsole::AfterLastFile(const CHashBundle &hb) +HRESULT CHashCallbackConsole::AfterLastFile(CHashBundle &hb) { ClosePercents2(); diff --git a/CPP/7zip/UI/Console/List.cpp b/CPP/7zip/UI/Console/List.cpp index ebcabb68..c56e2e21 100644 --- a/CPP/7zip/UI/Console/List.cpp +++ b/CPP/7zip/UI/Console/List.cpp @@ -1072,18 +1072,24 @@ HRESULT ListArchives(CCodecs *codecs, errorCode = ERROR_FILE_NOT_FOUND; lastError = HRESULT_FROM_WIN32(lastError);; g_StdOut.Flush(); - *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << endl; - g_ErrStream->NormalizePrint_UString(arcPath); - *g_ErrStream << endl << endl; + if (g_ErrStream) + { + *g_ErrStream << endl << kError << NError::MyFormatMessage(errorCode) << endl; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << endl << endl; + } numErrors++; continue; } if (fi.IsDir()) { g_StdOut.Flush(); - *g_ErrStream << endl << kError; - g_ErrStream->NormalizePrint_UString(arcPath); - *g_ErrStream << " is not a file" << endl << endl; + if (g_ErrStream) + { + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " is not a file" << endl << endl; + } numErrors++; continue; } @@ -1133,24 +1139,28 @@ HRESULT ListArchives(CCodecs *codecs, { if (result == E_ABORT) return result; - g_StdOut.Flush(); - *g_ErrStream << endl << kError; - g_ErrStream->NormalizePrint_UString(arcPath); - *g_ErrStream << " : "; - if (result == S_FALSE) - { - Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); - } - else - { + if (result != S_FALSE) lastError = result; - *g_ErrStream << "opening : "; - if (result == E_OUTOFMEMORY) - *g_ErrStream << "Can't allocate required memory"; + g_StdOut.Flush(); + if (g_ErrStream) + { + *g_ErrStream << endl << kError; + g_ErrStream->NormalizePrint_UString(arcPath); + *g_ErrStream << " : "; + if (result == S_FALSE) + { + Print_OpenArchive_Error(*g_ErrStream, codecs, arcLink); + } else - *g_ErrStream << NError::MyFormatMessage(result); + { + *g_ErrStream << "opening : "; + if (result == E_OUTOFMEMORY) + *g_ErrStream << "Can't allocate required memory"; + else + *g_ErrStream << NError::MyFormatMessage(result); + } + *g_ErrStream << endl; } - *g_ErrStream << endl; numErrors++; continue; } diff --git a/CPP/7zip/UI/Console/Main.cpp b/CPP/7zip/UI/Console/Main.cpp index 8f2825ca..4d7aac90 100644 --- a/CPP/7zip/UI/Console/Main.cpp +++ b/CPP/7zip/UI/Console/Main.cpp @@ -24,6 +24,7 @@ #include "../../../Windows/TimeUtils.h" #include "../Common/ArchiveCommandLine.h" +#include "../Common/Bench.h" #include "../Common/ExitCode.h" #include "../Common/Extract.h" @@ -56,8 +57,6 @@ using namespace NCommandLineParser; HINSTANCE g_hInstance = 0; #endif -extern bool g_LargePagesMode; - extern CStdOutStream *g_StdStream; extern CStdOutStream *g_ErrStream; @@ -88,7 +87,7 @@ static const char * const kHelpString = "a" #endif #endif - " [...] [...]\n" + " [...] [...] [@listfile]\n" "\n" "\n" " a : Add files to archive\n" @@ -104,8 +103,7 @@ static const char * const kHelpString = " x : eXtract files with full paths\n" "\n" "\n" - " -- : Stop switches parsing\n" - " @listfile : set path to listfile that contains file names\n" + " -- : Stop switches and @listfile parsing\n" " -ai[r[-|0]]{@listfile|!wildcard} : Include archives\n" " -ax[r[-|0]]{@listfile|!wildcard} : eXclude archives\n" " -ao{a|s|t|u} : set Overwrite mode\n" @@ -236,7 +234,7 @@ static void PrintWarningsPaths(const CErrorPathCodes &pc, CStdOutStream &so) { FOR_VECTOR(i, pc.Paths) { - so.NormalizePrint_UString(pc.Paths[i]); + so.NormalizePrint_UString(fs2us(pc.Paths[i])); so << " : "; so << NError::MyFormatMessage(pc.Codes[i]) << endl; } @@ -376,8 +374,13 @@ static void PrintMemUsage(const char *s, UInt64 val) *g_StdStream << " " << s << " Memory ="; PrintNum(SHIFT_SIZE_VALUE(val, 20), 7); *g_StdStream << " MB"; - if (g_LargePagesMode) - *g_StdStream << " (LP)"; + + #ifdef _7ZIP_LARGE_PAGES + AString lp; + Add_LargePages_String(lp); + if (!lp.IsEmpty()) + *g_StdStream << lp; + #endif } EXTERN_C_BEGIN @@ -911,7 +914,7 @@ int Main2( { hashCalc = &hb; ThrowException_if_Error(hb.SetMethods(EXTERNAL_CODECS_VARS_L options.HashMethods)); - hb.Init(); + // hb.Init(); } hresultMain = Extract( diff --git a/CPP/7zip/UI/Console/MainAr.cpp b/CPP/7zip/UI/Console/MainAr.cpp index fdac6434..1d6c5a42 100644 --- a/CPP/7zip/UI/Console/MainAr.cpp +++ b/CPP/7zip/UI/Console/MainAr.cpp @@ -2,6 +2,10 @@ #include "StdAfx.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif + #include "../../../Common/MyException.h" #include "../../../Common/StdOutStream.h" @@ -63,6 +67,10 @@ int MY_CDECL main try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif + res = Main2( #ifndef _WIN32 numArgs, args diff --git a/CPP/7zip/UI/Console/makefile b/CPP/7zip/UI/Console/makefile index 31bc5c28..2210e0b8 100644 --- a/CPP/7zip/UI/Console/makefile +++ b/CPP/7zip/UI/Console/makefile @@ -1,12 +1,7 @@ PROG = 7z.exe -MY_CONSOLE = 1 CFLAGS = $(CFLAGS) \ -DEXTERNAL_CODECS \ -!IFNDEF UNDER_CE -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -D_7ZIP_LARGE_PAGES -DSUPPORT_DEVICE_FILE -!ENDIF - COMMON_OBJS = \ $O\CommandLineParser.obj \ $O\CRC.obj \ @@ -57,7 +52,7 @@ AR_COMMON_OBJS = \ COMPRESS_OBJS = \ $O\CopyCoder.obj \ -C_OBJS = \ +C_OBJS = $(C_OBJS) \ $O\Alloc.obj \ $O\CpuArch.obj \ $O\Sort.obj \ diff --git a/CPP/7zip/UI/Explorer/ContextMenu.cpp b/CPP/7zip/UI/Explorer/ContextMenu.cpp index 59d78e5c..ea3b231c 100644 --- a/CPP/7zip/UI/Explorer/ContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/ContextMenu.cpp @@ -120,7 +120,11 @@ HRESULT CZipContextMenu::InitContextMenu(const wchar_t * /* folder */, const wch _isMenuForFM = true; _fileNames.Clear(); for (UInt32 i = 0; i < numFiles; i++) + { + // MessageBoxW(0, names[i], NULL, 0); + // OutputDebugStringW(names[i]); _fileNames.Add(names[i]); + } _dropMode = false; return S_OK; } @@ -225,13 +229,13 @@ static const CHashCommand g_HashCommands[] = { CZipContextMenu::kHash_CRC64, "CRC-64", "CRC64" }, { CZipContextMenu::kHash_XXH32, "XXH-32", "XXH32" }, { CZipContextMenu::kHash_XXH64, "XXH-64", "XXH64" }, - { CZipContextMenu::kHash_MD5, "MD2", "MD2" }, - { CZipContextMenu::kHash_MD5, "MD4", "MD4" }, + { CZipContextMenu::kHash_MD2, "MD2", "MD2" }, + { CZipContextMenu::kHash_MD4, "MD4", "MD4" }, { CZipContextMenu::kHash_MD5, "MD5", "MD5" }, { CZipContextMenu::kHash_SHA1, "SHA-1", "SHA1" }, { CZipContextMenu::kHash_SHA256, "SHA-256", "SHA256" }, - { CZipContextMenu::kHash_SHA256, "SHA-384", "SHA384" }, - { CZipContextMenu::kHash_SHA256, "SHA-512", "SHA512" }, + { CZipContextMenu::kHash_SHA384, "SHA-384", "SHA384" }, + { CZipContextMenu::kHash_SHA512, "SHA-512", "SHA512" }, { CZipContextMenu::kHash_BLAKE2sp, "BLAKE2sp", "BLAKE2sp" }, { CZipContextMenu::kHash_All, "*", "*" } }; @@ -436,6 +440,13 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, { // OutputDebugStringA("QueryContextMenu"); + /* + for (UInt32 i = 0; i < _fileNames.Size(); i++) + { + OutputDebugStringW(_fileNames[i]); + } + */ + LoadLangOneTime(); if (_fileNames.Size() == 0) return E_FAIL; @@ -595,7 +606,7 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } } - const UString &fileName = _fileNames.Front(); + // const UString &fileName = _fileNames.Front(); if (needExtract) { @@ -652,12 +663,8 @@ STDMETHODIMP CZipContextMenu::QueryContextMenu(HMENU hMenu, UINT indexMenu, } } - UString arcName; - if (_fileNames.Size() == 1) - arcName = CreateArchiveName(fi0, false); - else - arcName = CreateArchiveName(fileName, _fileNames.Size() > 1, false); - + const UString arcName = CreateArchiveName(_fileNames, _fileNames.Size() == 1 ? &fi0 : NULL); + UString arcName7z = arcName; arcName7z += ".7z"; UString arcNameZip = arcName; diff --git a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp index 72a3f1d5..9fc24659 100644 --- a/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp +++ b/CPP/7zip/UI/Explorer/DllExportsExplorer.cpp @@ -25,21 +25,17 @@ #include "ContextMenu.h" -static LPCTSTR const k_ShellExtName = TEXT("7-Zip-Zstandard Shell Extension"); +static LPCTSTR const k_ShellExtName = TEXT("7-Zip ZS Shell Extension"); static LPCTSTR const k_Approved = TEXT("Software\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved"); // {23170F69-40C1-278A-1000-000100020000} -#ifdef NEED_7ZIP_GUID -static LPCTSTR const k_Clsid = TEXT("{23170F69-40C1-278A-1000-000100020001}"); -#else -static LPCTSTR const k_Clsid = TEXT("{23170F69-0803-278A-1000-000100020001}"); -#endif +static LPCTSTR const k_Clsid = TEXT("{23170F69-20BB-278A-1000-000100020000}"); DEFINE_GUID(CLSID_CZipContextMenu, k_7zip_GUID_Data1, - k_7zip_GUID_Data2, + k_7zip_GUID_Data2_ZS, k_7zip_GUID_Data3_Common, - 0x10, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x01); + 0x10, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x00); using namespace NWindows; diff --git a/CPP/7zip/UI/Explorer/Explorer.dsp b/CPP/7zip/UI/Explorer/Explorer.dsp index 7840943f..9ce94b76 100644 --- a/CPP/7zip/UI/Explorer/Explorer.dsp +++ b/CPP/7zip/UI/Explorer/Explorer.dsp @@ -286,6 +286,15 @@ SOURCE=..\FileManager\RegistryUtils.h # PROP Default_Filter "" # Begin Source File +SOURCE=..\..\..\..\C\CpuArch.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\CpuArch.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Threads.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp b/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp index 58dd11d7..763a437f 100644 --- a/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp +++ b/CPP/7zip/UI/Explorer/RegistryContextMenu.cpp @@ -17,10 +17,10 @@ using namespace NRegistry; // CLISID (and Approved ?) items are separated for 32-bit and 64-bit code. // shellex items shared by 32-bit and 64-bit code? -#define k_Clsid_A "{23170F69-0803-278A-1000-000100020001}" +#define k_Clsid_A "{23170F69-20BB-278A-1000-000100020000}" static LPCTSTR const k_Clsid = TEXT(k_Clsid_A); -static LPCTSTR const k_ShellExtName = TEXT("7-Zip Shell Extension"); +static LPCTSTR const k_ShellExtName = TEXT("7-Zip ZS Shell Extension"); static LPCTSTR const k_Approved = TEXT("Software\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved"); static LPCTSTR const k_Inproc = TEXT("InprocServer32"); diff --git a/CPP/7zip/UI/Explorer/makefile b/CPP/7zip/UI/Explorer/makefile index f916e4e8..9ad709fc 100644 --- a/CPP/7zip/UI/Explorer/makefile +++ b/CPP/7zip/UI/Explorer/makefile @@ -68,6 +68,7 @@ FM_OBJS = \ $O\RegistryUtils.obj \ C_OBJS = \ + $O\CpuArch.obj \ $O\Threads.obj \ !include "../../7zip.mak" diff --git a/CPP/7zip/UI/Far/Far.cpp b/CPP/7zip/UI/Far/Far.cpp index 87aec40d..f4959cd4 100644 --- a/CPP/7zip/UI/Far/Far.cpp +++ b/CPP/7zip/UI/Far/Far.cpp @@ -327,7 +327,7 @@ HRESULT OpenArchive(const CSysString &fileName, } */ -static HANDLE MyOpenFilePluginW(const wchar_t *name) +static HANDLE MyOpenFilePluginW(const wchar_t *name, bool isAbortCodeSupported) { FString normalizedName = us2fs(name); normalizedName.Trim(); @@ -373,7 +373,12 @@ static HANDLE MyOpenFilePluginW(const wchar_t *name) archiverInfoResult, defaultName, openArchiveCallback); */ if (result == E_ABORT) - return (HANDLE)-2; + { + // fixed 18.06: + // OpenFilePlugin() is allowed to return (HANDLE)-2 as abort code + // OpenPlugin() is not allowed to return (HANDLE)-2. + return isAbortCodeSupported ? (HANDLE)-2 : INVALID_HANDLE_VALUE; + } UString errorMessage = agent->GetErrorMessage(); if (!errorMessage.IsEmpty()) @@ -403,7 +408,7 @@ static HANDLE MyOpenFilePluginW(const wchar_t *name) return (HANDLE)(plugin); } -static HANDLE MyOpenFilePlugin(const char *name) +static HANDLE MyOpenFilePlugin(const char *name, bool isAbortCodeSupported) { UINT codePage = #ifdef UNDER_CE @@ -411,7 +416,7 @@ static HANDLE MyOpenFilePlugin(const char *name) #else ::AreFileApisANSI() ? CP_ACP : CP_OEMCP; #endif - return MyOpenFilePluginW(GetUnicodeString(name, codePage)); + return MyOpenFilePluginW(GetUnicodeString(name, codePage), isAbortCodeSupported); } EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const unsigned char * /* data */, int /* dataSize */) @@ -423,7 +428,7 @@ EXTERN_C HANDLE WINAPI OpenFilePlugin(char *name, const unsigned char * /* data // if (!Opt.ProcessShiftF1) return(INVALID_HANDLE_VALUE); } - return MyOpenFilePlugin(name); + return MyOpenFilePlugin(name, true); // isAbortCodeSupported MY_TRY_END2("OpenFilePlugin", INVALID_HANDLE_VALUE); } @@ -458,7 +463,7 @@ EXTERN_C HANDLE WINAPI OpenPlugin(int openFrom, INT_PTR item) fileName.DeleteBack(); fileName.DeleteFrontal(1); } - return MyOpenFilePlugin(fileName); + return MyOpenFilePlugin(fileName, false); // isAbortCodeSupported } if (openFrom == OPEN_PLUGINSMENU) @@ -470,7 +475,7 @@ EXTERN_C HANDLE WINAPI OpenPlugin(int openFrom, INT_PTR item) PluginPanelItem pluginPanelItem; if (!g_StartupInfo.ControlGetActivePanelCurrentItemInfo(pluginPanelItem)) throw 142134; - return MyOpenFilePlugin(pluginPanelItem.FindData.cFileName); + return MyOpenFilePlugin(pluginPanelItem.FindData.cFileName, false); // isAbortCodeSupported } case 1: diff --git a/CPP/7zip/UI/FileManager/AboutDialog.rc b/CPP/7zip/UI/FileManager/AboutDialog.rc index edc83e58..a5cf4b8b 100644 --- a/CPP/7zip/UI/FileManager/AboutDialog.rc +++ b/CPP/7zip/UI/FileManager/AboutDialog.rc @@ -1,26 +1,27 @@ -#include -#include -#include #include "AboutDialogRes.h" -#include "..\..\GuiCommon.rc" -#include "..\..\MyVersion.h" +#include "../../GuiCommon.rc" +#include "../../MyVersion.h" -LANGUAGE LANG_ENGLISH,SUBLANG_ENGLISH_US +#define xc 216 +#define yc 144 -IDI_LOGO ICON "7zipLogo.ico" +#define y 93 -IDD_ABOUT DIALOG DISCARDABLE 0, 0, 224, 132 -STYLE WS_POPUP|DS_MODALFRAME|DS_CENTER|WS_CAPTION|WS_SYSMENU +IDI_LOGO ICON "../../UI/FileManager/7zipLogo.ico" + +#ifndef SS_REALSIZEIMAGE +#define SS_REALSIZEIMAGE 0x800 +#endif + +IDD_ABOUT DIALOG 0, 0, xs, ys MY_MODAL_DIALOG_STYLE MY_FONT CAPTION "About 7-Zip" -FONT 8, "MS Shell Dlg" { - CONTROL "OK", IDOK, "Button", BS_DEFPUSHBUTTON|WS_TABSTOP, 4, 112, 64, 16 - CONTROL "www.7-zip.org", IDB_ABOUT_HOMEPAGE, "Button", WS_TABSTOP, 72, 112, 64, 16 - CONTROL IDI_LOGO, -1, "Static", SS_ICON|SS_REALSIZEIMAGE, 8, 8, 32, 32 - CONTROL "", IDT_ABOUT_VERSION, "Static", WS_GROUP, 8, 54, 208, 8 - CONTROL "", IDT_ABOUT_DATE, "Static", WS_GROUP, 8, 67, 208, 8 - CONTROL MY_COPYRIGHT_CR, -1, "Static", WS_GROUP, 8, 80, 208, 8 - CONTROL "7-Zip is free software", IDT_ABOUT_INFO, "Static", WS_GROUP, 8, 93, 208, 19 - CONTROL "7-Zip ZS Homepage", IDB_ABOUT_HOMEPAGE2, "Button", WS_TABSTOP, 140, 112, 76, 16 + DEFPUSHBUTTON "OK", IDOK, bx3-10, by, bxs, bys + PUSHBUTTON "www.7-zip.org", IDB_ABOUT_HOMEPAGE, bx2-10, by, bxs, bys + PUSHBUTTON "7-Zip ZS Hompeage", IDB_ABOUT_HOMEPAGE2, bx1-10, by, bxs+10, bys + ICON IDI_LOGO, -1, m, m, 32, 32, SS_REALSIZEIMAGE, + LTEXT "", IDT_ABOUT_VERSION, m, 54, xc, 8 + LTEXT "", IDT_ABOUT_DATE, m, 67, xc, 8 + LTEXT MY_COPYRIGHT_CR, -1, m, 80, xc, 8 + LTEXT "7-Zip is free software", IDT_ABOUT_INFO, m, y, xc, (by - y - 1) } - diff --git a/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp b/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp index 70915518..50118b47 100644 --- a/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp +++ b/CPP/7zip/UI/FileManager/AltStreamsFolder.cpp @@ -468,7 +468,7 @@ static HRESULT UpdateFile(NFsFolder::CCopyStateIO &state, CFSTR inPath, CFSTR ou { if (NFind::DoesFileOrDirExist(outPath)) { - RINOK(SendMessageError(callback, NError::MyFormatMessage(ERROR_ALREADY_EXISTS), outPath)); + RINOK(SendMessageError(callback, NError::MyFormatMessage(ERROR_ALREADY_EXISTS), FString(outPath))); CFileInfo fi; if (fi.Find(inPath)) { diff --git a/CPP/7zip/UI/FileManager/ExtractCallback.cpp b/CPP/7zip/UI/FileManager/ExtractCallback.cpp index 6433e917..f2305944 100644 --- a/CPP/7zip/UI/FileManager/ExtractCallback.cpp +++ b/CPP/7zip/UI/FileManager/ExtractCallback.cpp @@ -759,11 +759,15 @@ STDMETHODIMP CExtractCallbackImp::AskWrite( destPathResultTemp = fs2us(destPathSys); } else + { + if (NFind::DoesFileExist(destPathSys)) if (!NDir::DeleteFileAlways(destPathSys)) + if (GetLastError() != ERROR_FILE_NOT_FOUND) { RINOK(MessageError("can not delete output file", destPathSys)); return E_ABORT; } + } } *writeAnswer = BoolToInt(true); return StringToBstr(destPathResultTemp, destPathResult); diff --git a/CPP/7zip/UI/FileManager/FM.cpp b/CPP/7zip/UI/FileManager/FM.cpp index 43e45bb0..c89e9e78 100644 --- a/CPP/7zip/UI/FileManager/FM.cpp +++ b/CPP/7zip/UI/FileManager/FM.cpp @@ -7,6 +7,9 @@ #include #include "../../../../C/Alloc.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif #include "../../../Common/StringConvert.h" #include "../../../Common/StringToInt.h" @@ -660,6 +663,9 @@ int WINAPI WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, { try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif return WinMain2(nCmdShow); } catch (...) diff --git a/CPP/7zip/UI/FileManager/FM.dsp b/CPP/7zip/UI/FileManager/FM.dsp index 1e3b48a0..3c51f6a5 100644 --- a/CPP/7zip/UI/FileManager/FM.dsp +++ b/CPP/7zip/UI/FileManager/FM.dsp @@ -712,10 +712,24 @@ SOURCE=..\..\..\..\C\Alloc.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\CpuArch.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c # SUBTRACT CPP /YX /Yc /Yu # End Source File diff --git a/CPP/7zip/UI/FileManager/FM.mak b/CPP/7zip/UI/FileManager/FM.mak index 4838dc17..493b4bbd 100644 --- a/CPP/7zip/UI/FileManager/FM.mak +++ b/CPP/7zip/UI/FileManager/FM.mak @@ -1,3 +1,16 @@ +CFLAGS = $(CFLAGS) \ + -DLANG \ + -DNEW_FOLDER_INTERFACE \ + +!IFDEF UNDER_CE +LIBS = $(LIBS) ceshell.lib Commctrl.lib +!ELSE +LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib +CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE +LFLAGS = $(LFLAGS) /DELAYLOAD:mpr.dll +LIBS = $(LIBS) delayimp.lib +!ENDIF + FM_OBJS = \ $O\App.obj \ $O\BrowseDialog.obj \ @@ -73,6 +86,9 @@ WIN_OBJS = $(WIN_OBJS) \ !ENDIF +C_OBJS = $(C_OBJS) \ + $O\DllSecur.obj \ + AGENT_OBJS = \ $O\Agent.obj \ $O\AgentOut.obj \ diff --git a/CPP/7zip/UI/FileManager/FSFolder.cpp b/CPP/7zip/UI/FileManager/FSFolder.cpp index a6396c79..4fcec45f 100644 --- a/CPP/7zip/UI/FileManager/FSFolder.cpp +++ b/CPP/7zip/UI/FileManager/FSFolder.cpp @@ -532,9 +532,8 @@ STDMETHODIMP CFSFolder::GetNumRawProps(UInt32 *numProps) return S_OK; } -STDMETHODIMP CFSFolder::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID) +STDMETHODIMP CFSFolder::GetRawPropInfo(UInt32 /* index */, BSTR *name, PROPID *propID) { - index = index; *name = NULL; *propID = kpidNtReparse; return S_OK; diff --git a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp index cc06aef1..5be17faa 100644 --- a/CPP/7zip/UI/FileManager/FSFolderCopy.cpp +++ b/CPP/7zip/UI/FileManager/FSFolderCopy.cpp @@ -161,8 +161,6 @@ static DWORD CALLBACK CopyProgressRoutine( LPVOID lpData // from CopyFileEx ) { - TotalFileSize = TotalFileSize; - // TotalBytesTransferred = TotalBytesTransferred; // StreamSize = StreamSize; // StreamBytesTransferred = StreamBytesTransferred; // dwStreamNumber = dwStreamNumber; diff --git a/CPP/7zip/UI/FileManager/ListViewDialog.cpp b/CPP/7zip/UI/FileManager/ListViewDialog.cpp index 79f566f2..c5b0d438 100644 --- a/CPP/7zip/UI/FileManager/ListViewDialog.cpp +++ b/CPP/7zip/UI/FileManager/ListViewDialog.cpp @@ -213,6 +213,12 @@ void CListViewDialog::ShowItemInfo() if (index < Values.Size()) dlg.Text = Values[index]; } + + #ifdef _WIN32 + if (dlg.Text.Find(L'\r') < 0) + dlg.Text.Replace(L"\n", L"\r\n"); + #endif + dlg.Create(*this); } diff --git a/CPP/7zip/UI/FileManager/Panel.cpp b/CPP/7zip/UI/FileManager/Panel.cpp index 8e126317..8f1c8e50 100644 --- a/CPP/7zip/UI/FileManager/Panel.cpp +++ b/CPP/7zip/UI/FileManager/Panel.cpp @@ -879,8 +879,9 @@ void CPanel::AddToArchive() FOR_VECTOR (i, indices) names.Add(curPrefix + GetItemRelPath2(indices[i])); - bool fromPrev = (names.Size() > 1); - const UString arcName = CreateArchiveName(names.Front(), fromPrev, false); + + const UString arcName = CreateArchiveName(names); + HRESULT res = CompressFiles(destCurDirPrefix, arcName, L"", true, // addExtension names, false, true, false); diff --git a/CPP/7zip/UI/FileManager/PanelCopy.cpp b/CPP/7zip/UI/FileManager/PanelCopy.cpp index bc69db64..e0640eff 100644 --- a/CPP/7zip/UI/FileManager/PanelCopy.cpp +++ b/CPP/7zip/UI/FileManager/PanelCopy.cpp @@ -29,7 +29,7 @@ public: CMyComPtr ExtractCallback; CHashBundle Hash; - UString FirstFilePath; + // UString FirstFilePath; HRESULT Result; @@ -48,7 +48,7 @@ void CPanelCopyThread::ShowFinalResults(HWND hwnd) if (!ResultsWereShown) { ResultsWereShown = true; - ShowHashResults(Hash, fs2us(FirstFilePath), hwnd); + ShowHashResults(Hash, hwnd); } } @@ -100,7 +100,7 @@ HRESULT CPanelCopyThread::ProcessVirt() else if (options->testMode) { CProgressMessageBoxPair &pair = GetMessagePair(false); // GetMessagePair(ExtractCallbackSpec->Hash.NumErrors != 0); - AddHashBundleRes(pair.Message, Hash, FirstFilePath); + AddHashBundleRes(pair.Message, Hash); } } @@ -158,7 +158,10 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind if (indices.Size() == 1) - extracter.FirstFilePath = GetItemRelPath(indices[0]); + { + extracter.Hash.FirstFileName = GetItemRelPath(indices[0]); + extracter.Hash.MainName = extracter.Hash.FirstFileName; + } if (options.VirtFileSystem) { @@ -186,7 +189,7 @@ HRESULT CPanel::CopyTo(CCopyToOptions &options, const CRecordVector &ind extracter.ExtractCallbackSpec->SetHashCalc(&extracter.Hash); } - extracter.Hash.Init(); + // extracter.Hash.Init(); UString title; { diff --git a/CPP/7zip/UI/FileManager/PanelCrc.cpp b/CPP/7zip/UI/FileManager/PanelCrc.cpp index 6353a722..94c41e97 100644 --- a/CPP/7zip/UI/FileManager/PanelCrc.cpp +++ b/CPP/7zip/UI/FileManager/PanelCrc.cpp @@ -147,7 +147,7 @@ class CThreadCrc: public CProgressThreadVirt public: CDirEnumerator Enumerator; CHashBundle Hash; - FString FirstFilePath; + // FString FirstFilePath; void SetStatus(const UString &s); void AddErrorMessage(DWORD systemError, const FChar *name); @@ -165,7 +165,7 @@ void CThreadCrc::ShowFinalResults(HWND hwnd) if (!ResultsWereShown) { ResultsWereShown = true; - ShowHashResults(Hash, fs2us(FirstFilePath), hwnd); + ShowHashResults(Hash, hwnd); } } @@ -193,7 +193,7 @@ void CThreadCrc::SetStatus(const UString &s2) HRESULT CThreadCrc::ProcessVirt() { - Hash.Init(); + // Hash.Init(); CMyBuffer buf; if (!buf.Allocate(kBufSize)) @@ -289,7 +289,7 @@ HRESULT CThreadCrc::ProcessVirt() } if (isFirstFile) { - FirstFilePath = path; + Hash.FirstFileName = path; isFirstFile = false; } sync.Set_FilePath(fs2us(path)); @@ -370,9 +370,13 @@ HRESULT CApp::CalculateCrc2(const UString &methodName) methods.Add(methodName); RINOK(t.Hash.SetMethods(EXTERNAL_CODECS_VARS_G methods)); } + FOR_VECTOR (i, indices) t.Enumerator.FilePaths.Add(us2fs(srcPanel.GetItemRelPath(indices[i]))); + if (t.Enumerator.FilePaths.Size() == 1) + t.Hash.MainName = t.Enumerator.FilePaths[0]; + UString basePrefix = srcPanel.GetFsPath(); UString basePrefix2 = basePrefix; if (basePrefix2.Back() == ':') diff --git a/CPP/7zip/UI/FileManager/PanelDrag.cpp b/CPP/7zip/UI/FileManager/PanelDrag.cpp index f28d3eb0..ee4a9f6f 100644 --- a/CPP/7zip/UI/FileManager/PanelDrag.cpp +++ b/CPP/7zip/UI/FileManager/PanelDrag.cpp @@ -940,8 +940,10 @@ void CPanel::CompressDropFiles(const UStringVector &fileNames, const UString &fo if (IsFolderInTemp(folderPath2F)) folderPath2 = ROOT_FS_FOLDER; } - const UString archiveName = CreateArchiveName(fileNames.Front(), (fileNames.Size() > 1), false); - CompressFiles(folderPath2, archiveName, L"", + + const UString arcName = CreateArchiveName(fileNames); + + CompressFiles(folderPath2, arcName, L"", true, // addExtension fileNames, false, // email diff --git a/CPP/7zip/UI/FileManager/makefile b/CPP/7zip/UI/FileManager/makefile index 8b69a67b..27dce9fd 100644 --- a/CPP/7zip/UI/FileManager/makefile +++ b/CPP/7zip/UI/FileManager/makefile @@ -1,16 +1,7 @@ PROG = 7zFM.exe CFLAGS = $(CFLAGS) \ - -DLANG \ - -DNEW_FOLDER_INTERFACE \ -DEXTERNAL_CODECS \ -!IFDEF UNDER_CE -LIBS = $(LIBS) ceshell.lib Commctrl.lib -!ELSE -LIBS = $(LIBS) comctl32.lib htmlhelp.lib comdlg32.lib Mpr.lib Gdi32.lib -CFLAGS = $(CFLAGS) -DWIN_LONG_PATH -DSUPPORT_DEVICE_FILE -!ENDIF - !include "FM.mak" COMMON_OBJS = \ @@ -102,8 +93,9 @@ GUI_OBJS = \ COMPRESS_OBJS = \ $O\CopyCoder.obj \ -C_OBJS = \ +C_OBJS = $(C_OBJS) \ $O\Alloc.obj \ + $O\CpuArch.obj \ $O\Sort.obj \ $O\Threads.obj \ diff --git a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp index fba782b0..f8af962a 100644 --- a/CPP/7zip/UI/GUI/BenchmarkDialog.cpp +++ b/CPP/7zip/UI/GUI/BenchmarkDialog.cpp @@ -324,7 +324,6 @@ static void PrintSize_MB(UString &s, UInt64 size) s += kMB; } -extern bool g_LargePagesMode; UInt32 CBenchmarkDialog::OnChangeDictionary() { @@ -338,8 +337,14 @@ UInt32 CBenchmarkDialog::OnChangeDictionary() s += " / "; PrintSize_MB(s, ramSize); } - if (g_LargePagesMode) - s += " LP"; + + #ifdef _7ZIP_LARGE_PAGES + { + AString s2; + Add_LargePages_String(s2); + s += s2; + } + #endif SetItemText(IDT_BENCH_MEMORY_VAL, s); diff --git a/CPP/7zip/UI/GUI/ExtractGUI.cpp b/CPP/7zip/UI/GUI/ExtractGUI.cpp index 37aa45b3..99db743d 100644 --- a/CPP/7zip/UI/GUI/ExtractGUI.cpp +++ b/CPP/7zip/UI/GUI/ExtractGUI.cpp @@ -99,8 +99,10 @@ HRESULT CThreadExtracting::ProcessVirt() CDecompressStat Stat; #ifndef _SFX + /* if (HashBundle) HashBundle->Init(); + */ #endif HRESULT res = Extract(codecs, @@ -119,7 +121,7 @@ HRESULT CThreadExtracting::ProcessVirt() { AddValuePair(Pairs, IDS_ARCHIVES_COLON, Stat.NumArchives); AddSizeValuePair(Pairs, IDS_PROP_PACKED_SIZE, Stat.PackSize); - AddHashBundleRes(Pairs, *HashBundle, UString()); + AddHashBundleRes(Pairs, *HashBundle); } else if (Options->TestMode) { diff --git a/CPP/7zip/UI/GUI/GUI.cpp b/CPP/7zip/UI/GUI/GUI.cpp index f613f97d..b58e4bd2 100644 --- a/CPP/7zip/UI/GUI/GUI.cpp +++ b/CPP/7zip/UI/GUI/GUI.cpp @@ -2,6 +2,10 @@ #include "StdAfx.h" +#ifdef _WIN32 +#include "../../../../C/DllSecur.h" +#endif + #include "../../../Common/MyWindows.h" #include @@ -372,6 +376,10 @@ int APIENTRY WinMain(HINSTANCE hInstance, HINSTANCE /* hPrevInstance */, // setlocale(LC_COLLATE, ".ACP"); try { + #ifdef _WIN32 + My_SetDefaultDllDirectories(); + #endif + return Main2(); } catch(const CNewException &) diff --git a/CPP/7zip/UI/GUI/GUI.dsp b/CPP/7zip/UI/GUI/GUI.dsp index e9aaa889..70448ff2 100644 --- a/CPP/7zip/UI/GUI/GUI.dsp +++ b/CPP/7zip/UI/GUI/GUI.dsp @@ -810,6 +810,15 @@ SOURCE=..\..\..\..\C\CpuArch.h # End Source File # Begin Source File +SOURCE=..\..\..\..\C\DllSecur.c +# SUBTRACT CPP /YX /Yc /Yu +# End Source File +# Begin Source File + +SOURCE=..\..\..\..\C\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\..\..\C\Sort.c !IF "$(CFG)" == "GUI - Win32 Release" diff --git a/CPP/7zip/UI/GUI/HashGUI.cpp b/CPP/7zip/UI/GUI/HashGUI.cpp index 5aef24c1..7c0b3f28 100644 --- a/CPP/7zip/UI/GUI/HashGUI.cpp +++ b/CPP/7zip/UI/GUI/HashGUI.cpp @@ -27,6 +27,7 @@ class CHashCallbackGUI: public CProgressThreadVirt, public IHashCallbackUI UInt64 NumFiles; bool _curIsFolder; UString FirstFileName; + // UString MainPath; CPropNameValPairs PropNameValPairs; @@ -71,9 +72,9 @@ void AddSizeValue(UString &s, UInt64 value) if (value >= (1 << 10)) { char c; - if (value >= ((UInt64)10 << 30)) { value >>= 30; c = 'G'; } - if (value >= (10 << 20)) { value >>= 20; c = 'M'; } - else { value >>= 10; c = 'K'; } + if (value >= ((UInt64)10 << 30)) { value >>= 30; c = 'G'; } + else if (value >= (10 << 20)) { value >>= 20; c = 'M'; } + else { value >>= 10; c = 'K'; } char sz[32]; ConvertUInt64ToString(value, sz); s += " ("; @@ -192,19 +193,25 @@ static void AddHashResString(CPropNameValPairs &s, const CHasherState &h, unsign } -void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb, const UString &firstFileName) +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb) { if (hb.NumErrors != 0) AddValuePair(s, IDS_PROP_NUM_ERRORS, hb.NumErrors); - - if (hb.NumFiles == 1 && hb.NumDirs == 0 && !firstFileName.IsEmpty()) + + if (hb.NumFiles == 1 && hb.NumDirs == 0 && !hb.FirstFileName.IsEmpty()) { CProperty &pair = s.AddNew(); LangString(IDS_PROP_NAME, pair.Name); - pair.Value = firstFileName; + pair.Value = hb.FirstFileName; } else { + if (!hb.MainName.IsEmpty()) + { + CProperty &pair = s.AddNew(); + LangString(IDS_PROP_NAME, pair.Name); + pair.Value = hb.MainName; + } if (hb.NumDirs != 0) AddValuePair(s, IDS_PROP_FOLDERS, hb.NumDirs); AddValuePair(s, IDS_PROP_FILES, hb.NumFiles); @@ -240,10 +247,10 @@ void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb, const UString } -void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName) +void AddHashBundleRes(UString &s, const CHashBundle &hb) { CPropNameValPairs pairs; - AddHashBundleRes(pairs, hb, firstFileName); + AddHashBundleRes(pairs, hb); FOR_VECTOR (i, pairs) { @@ -263,9 +270,11 @@ void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFil } -HRESULT CHashCallbackGUI::AfterLastFile(const CHashBundle &hb) +HRESULT CHashCallbackGUI::AfterLastFile(CHashBundle &hb) { - AddHashBundleRes(PropNameValPairs, hb, FirstFileName); + hb.FirstFileName = FirstFileName; + // MainPath + AddHashBundleRes(PropNameValPairs, hb); CProgressSync &sync = Sync; sync.Set_NumFilesCur(hb.NumFiles); @@ -335,10 +344,10 @@ void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd) } -void ShowHashResults(const CHashBundle &hb, const UString &firstFileName, HWND hwnd) +void ShowHashResults(const CHashBundle &hb, HWND hwnd) { CPropNameValPairs propPairs; - AddHashBundleRes(propPairs, hb, firstFileName); + AddHashBundleRes(propPairs, hb); ShowHashResults(propPairs, hwnd); } diff --git a/CPP/7zip/UI/GUI/HashGUI.h b/CPP/7zip/UI/GUI/HashGUI.h index d6caa53e..b6268232 100644 --- a/CPP/7zip/UI/GUI/HashGUI.h +++ b/CPP/7zip/UI/GUI/HashGUI.h @@ -18,10 +18,10 @@ void AddValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); void AddSizeValue(UString &s, UInt64 value); void AddSizeValuePair(CPropNameValPairs &pairs, UINT resourceID, UInt64 value); -void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb, const UString &firstFileName); -void AddHashBundleRes(UString &s, const CHashBundle &hb, const UString &firstFileName); +void AddHashBundleRes(CPropNameValPairs &s, const CHashBundle &hb); +void AddHashBundleRes(UString &s, const CHashBundle &hb); void ShowHashResults(const CPropNameValPairs &propPairs, HWND hwnd); -void ShowHashResults(const CHashBundle &hb, const UString &firstFileName, HWND hwnd); +void ShowHashResults(const CHashBundle &hb, HWND hwnd); #endif diff --git a/CPP/7zip/UI/GUI/makefile b/CPP/7zip/UI/GUI/makefile index 088da0aa..c608acdc 100644 --- a/CPP/7zip/UI/GUI/makefile +++ b/CPP/7zip/UI/GUI/makefile @@ -134,6 +134,7 @@ COMPRESS_OBJS = \ C_OBJS = \ $O\Alloc.obj \ $O\CpuArch.obj \ + $O\DllSecur.obj \ $O\Sort.obj \ $O\Threads.obj \ diff --git a/CPP/Build.mak b/CPP/Build.mak index b1c1c569..f2615788 100644 --- a/CPP/Build.mak +++ b/CPP/Build.mak @@ -5,16 +5,19 @@ CFLAGS = $(CFLAGS) -DUNICODE -D_UNICODE !ENDIF !IFNDEF O -!IFDEF CPU -O=$(CPU) +!IFDEF PLATFORM +O=$(PLATFORM) !ELSE -O=O +O=o !ENDIF !ENDIF -!IF "$(CPU)" == "AMD64" +# CFLAGS = $(CFLAGS) -FAsc -Fa$O/asm/ + + +!IF "$(PLATFORM)" == "x64" MY_ML = ml64 -Dx64 -WX -!ELSEIF "$(CPU)" == "ARM" +!ELSEIF "$(PLATFORM)" == "arm" MY_ML = armasm -WX !ELSE MY_ML = ml -WX @@ -27,16 +30,16 @@ RFLAGS = $(RFLAGS) -dUNDER_CE LFLAGS = $(LFLAGS) /ENTRY:mainACRTStartup !ENDIF !ELSE -!IFNDEF NEW_COMPILER -LFLAGS = $(LFLAGS) +!IFDEF OLD_COMPILER +LFLAGS = $(LFLAGS) -OPT:NOWIN98 !ENDIF -!IF "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" +!IF "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" CFLAGS = $(CFLAGS) -Gr !ENDIF LIBS = $(LIBS) user32.lib advapi32.lib shell32.lib !ENDIF -!IF "$(CPU)" == "ARM" +!IF "$(PLATFORM)" == "arm" COMPL_ASM = $(MY_ML) $** $O/$(*B).obj !ELSE COMPL_ASM = $(MY_ML) -c -Fo$O/ $** @@ -44,19 +47,19 @@ COMPL_ASM = $(MY_ML) -c -Fo$O/ $** CFLAGS = $(CFLAGS) -nologo -c -Fo$O/ -W4 -WX -EHsc -Gy -GR- -GF -GL -!IFDEF MY_STATIC_LINK +!IFDEF MY_DYNAMIC_LINK +CFLAGS = $(CFLAGS) -MD +!ELSE !IFNDEF MY_SINGLE_THREAD CFLAGS = $(CFLAGS) -MT !ENDIF -!ELSE -CFLAGS = $(CFLAGS) -MD !ENDIF -!IFDEF NEW_COMPILER +!IFNDEF OLD_COMPILER CFLAGS = $(CFLAGS) -GS- -Zc:forScope -Zc:wchar_t !IFNDEF UNDER_CE CFLAGS = $(CFLAGS) -MP2 -!IFNDEF CPU +!IFNDEF PLATFORM # CFLAGS = $(CFLAGS) -arch:IA32 !ENDIF !ENDIF @@ -64,7 +67,17 @@ CFLAGS = $(CFLAGS) -MP2 CFLAGS = $(CFLAGS) !ENDIF -!IF "$(CPU)" == "AMD64" +!IFDEF MY_CONSOLE +CFLAGS = $(CFLAGS) -D_CONSOLE +!ENDIF + +!IFNDEF UNDER_CE +!IF "$(PLATFORM)" == "arm" +CFLAGS = $(CFLAGS) -D_ARM_WINAPI_PARTITION_DESKTOP_SDK_AVAILABLE +!ENDIF +!ENDIF + +!IF "$(PLATFORM)" == "x64" CFLAGS_O1 = $(CFLAGS) -O1 !ELSE CFLAGS_O1 = $(CFLAGS) -O1 @@ -80,7 +93,7 @@ LFLAGS = $(LFLAGS) /LTCG /LARGEADDRESSAWARE !IFDEF DEF_FILE LFLAGS = $(LFLAGS) -DLL -DEF:$(DEF_FILE) !ELSE -!IF defined(MY_FIXED) && "$(CPU)" != "ARM" && "$(CPU)" != "ARM64" +!IF defined(MY_FIXED) && "$(PLATFORM)" != "arm" && "$(PLATFORM)" != "arm64" LFLAGS = $(LFLAGS) /FIXED !ELSE LFLAGS = $(LFLAGS) /FIXED:NO @@ -89,7 +102,7 @@ LFLAGS = $(LFLAGS) /FIXED:NO !ENDIF -# !IF "$(CPU)" == "AMD64" +# !IF "$(PLATFORM)" == "x64" !IFDEF SUB_SYS_VER @@ -129,10 +142,10 @@ clean: $O: if not exist "$O" mkdir "$O" -$O/Asm: - if not exist "$O/Asm" mkdir "$O/Asm" +$O/asm: + if not exist "$O/asm" mkdir "$O/asm" -$(PROGPATH): $O $O/Asm $(OBJS) $(DEF_FILE) +$(PROGPATH): $O $O/asm $(OBJS) $(DEF_FILE) link $(LFLAGS) -out:$(PROGPATH) $(OBJS) $(LIBS) !IFNDEF NO_DEFAULT_RES diff --git a/CPP/Common/ListFileUtils.cpp b/CPP/Common/ListFileUtils.cpp index f22680b5..2fdde6dd 100644 --- a/CPP/Common/ListFileUtils.cpp +++ b/CPP/Common/ListFileUtils.cpp @@ -25,14 +25,21 @@ static void AddName(UStringVector &strings, UString &s) strings.Add(s); } -bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage) +bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, DWORD &lastError) { + lastError = 0; NWindows::NFile::NIO::CInFile file; if (!file.Open(fileName)) + { + lastError = ::GetLastError(); return false; + } UInt64 fileSize; if (!file.GetLength(fileSize)) + { + lastError = ::GetLastError(); return false; + } if (fileSize >= ((UInt32)1 << 31) - 32) return false; UString u; @@ -43,7 +50,10 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage CByteArr buf((size_t)fileSize); UInt32 processed; if (!file.Read(buf, (UInt32)fileSize, processed)) + { + lastError = ::GetLastError(); return false; + } if (processed != fileSize) return false; file.Close(); @@ -74,7 +84,10 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage char *p = s.GetBuf((unsigned)fileSize); UInt32 processed; if (!file.Read(p, (UInt32)fileSize, processed)) + { + lastError = ::GetLastError(); return false; + } if (processed != fileSize) return false; file.Close(); @@ -94,7 +107,7 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage } const wchar_t kGoodBOM = 0xFEFF; - const wchar_t kBadBOM = 0xFFFE; + // const wchar_t kBadBOM = 0xFFFE; UString s; unsigned i = 0; @@ -102,8 +115,10 @@ bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage for (; i < u.Len(); i++) { wchar_t c = u[i]; + /* if (c == kGoodBOM || c == kBadBOM) return false; + */ if (c == '\n' || c == 0xD) { AddName(strings, s); diff --git a/CPP/Common/ListFileUtils.h b/CPP/Common/ListFileUtils.h index ec32d8e0..a4f0d167 100644 --- a/CPP/Common/ListFileUtils.h +++ b/CPP/Common/ListFileUtils.h @@ -9,6 +9,10 @@ #define MY__CP_UTF16 1200 #define MY__CP_UTF16BE 1201 -bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage = CP_OEMCP); +// bool ReadNamesFromListFile(CFSTR fileName, UStringVector &strings, UINT codePage = CP_OEMCP); + + // = CP_OEMCP +bool ReadNamesFromListFile2(CFSTR fileName, UStringVector &strings, UINT codePage, + DWORD &lastError); #endif diff --git a/CPP/Common/MyBuffer2.h b/CPP/Common/MyBuffer2.h index 5cabd73c..10edcb1c 100644 --- a/CPP/Common/MyBuffer2.h +++ b/CPP/Common/MyBuffer2.h @@ -5,7 +5,7 @@ #include "../../C/Alloc.h" -#include "Defs.h" +#include "MyTypes.h" class CMidBuffer { @@ -15,7 +15,7 @@ class CMidBuffer CLASS_NO_COPY(CMidBuffer) public: - CMidBuffer(): _data(NULL), _size(0) {}; + CMidBuffer(): _data(NULL), _size(0) {} ~CMidBuffer() { ::MidFree(_data); } void Free() { ::MidFree(_data); _data = NULL; _size = 0; } @@ -29,12 +29,12 @@ public: { if (!_data || size > _size) { + ::MidFree(_data); const size_t kMinSize = (size_t)1 << 16; if (size < kMinSize) size = kMinSize; - ::MidFree(_data); _size = 0; - _data = 0; + _data = NULL; _data = (Byte *)::MidAlloc(size); if (_data) _size = size; @@ -42,4 +42,59 @@ public: } }; + +class CAlignedBuffer +{ + Byte *_data; + size_t _size; + + CLASS_NO_COPY(CAlignedBuffer) + +public: + CAlignedBuffer(): _data(NULL), _size(0) {} + ~CAlignedBuffer() + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + } + + void Free() + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _data = NULL; + _size = 0; + } + + bool IsAllocated() const { return _data != NULL; } + operator Byte *() { return _data; } + operator const Byte *() const { return _data; } + size_t Size() const { return _size; } + + void Alloc(size_t size) + { + if (!_data || size != _size) + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _size = 0; + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (_data) + _size = size; + } + } + + void AllocAtLeast(size_t size) + { + if (!_data || size > _size) + { + ISzAlloc_Free(&g_AlignedAlloc, _data); + _size = 0; + _data = NULL; + _data = (Byte *)ISzAlloc_Alloc(&g_AlignedAlloc, size); + if (_data) + _size = size; + } + } +}; + + #endif diff --git a/CPP/Common/MyString.h b/CPP/Common/MyString.h index f484ad22..45cea98b 100644 --- a/CPP/Common/MyString.h +++ b/CPP/Common/MyString.h @@ -307,6 +307,7 @@ public: void ReplaceOneCharAtPos(unsigned pos, char c) { _chars[pos] = c; } + char *GetBuf() { return _chars; } /* GetBuf(minLen): provides the buffer that can store at least (minLen) characters and additional null terminator. 9.35: GetBuf doesn't preserve old characters and terminator */ diff --git a/CPP/Windows/MemoryLock.cpp b/CPP/Windows/MemoryLock.cpp index 3cd82e50..d1b3bcc3 100644 --- a/CPP/Windows/MemoryLock.cpp +++ b/CPP/Windows/MemoryLock.cpp @@ -2,6 +2,8 @@ #include "StdAfx.h" +#include "../../C/CpuArch.h" + #include "MemoryLock.h" namespace NWindows { @@ -75,6 +77,9 @@ typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); We suppose that Window 10 works incorrectly with "Large Pages" at: - Windows 10 1703 (15063) - Windows 10 1709 (16299) + + - Windows 10 1809 (17763) on some CPUs that have no 1 GB page support. + We need more information about that new BUG in Windows. */ unsigned Get_LargePages_RiskLevel() @@ -87,9 +92,19 @@ unsigned Get_LargePages_RiskLevel() if (!func) return 0; func(&vi); - return (vi.dwPlatformId == VER_PLATFORM_WIN32_NT - && vi.dwMajorVersion + vi.dwMinorVersion == 10 - && vi.dwBuildNumber <= 16299) ? 1 : 0; + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + return 0; + if (vi.dwMajorVersion + vi.dwMinorVersion != 10) + return 0; + if (vi.dwBuildNumber <= 16299) + return 1; + + #ifdef MY_CPU_X86_OR_AMD64 + if (!CPU_IsSupported_PageGB()) + return 1; + #endif + + return 0; } #endif diff --git a/CPP/Windows/Shell.cpp b/CPP/Windows/Shell.cpp index fde29ac7..7ba82d42 100644 --- a/CPP/Windows/Shell.cpp +++ b/CPP/Windows/Shell.cpp @@ -2,6 +2,11 @@ #include "StdAfx.h" +/* +#include +#include +*/ + #include "../Common/MyCom.h" #ifndef _UNICODE #include "../Common/StringConvert.h" @@ -114,9 +119,22 @@ UString CDrop::QueryFileName(UINT fileIndex) void CDrop::QueryFileNames(UStringVector &fileNames) { UINT numFiles = QueryCountOfFiles(); + /* + char s[100]; + sprintf(s, "QueryFileNames: %d files", numFiles); + OutputDebugStringA(s); + */ fileNames.ClearAndReserve(numFiles); for (UINT i = 0; i < numFiles; i++) - fileNames.AddInReserved(QueryFileName(i)); + { + const UString s2 = QueryFileName(i); + if (!s2.IsEmpty()) + fileNames.AddInReserved(s2); + /* + OutputDebugStringW(L"file ---"); + OutputDebugStringW(s2); + */ + } } diff --git a/CPP/appveyor.cmd b/CPP/appveyor.cmd index 14345aca..99b96fc1 100644 --- a/CPP/appveyor.cmd +++ b/CPP/appveyor.cmd @@ -1,10 +1,10 @@ @echo off -REM Microsoft Windows SDK 7.1 (VC=sdk71) +REM Microsoft Windows SDK 7.1 (VC=sdk71) -> can compile for IA64, but who needs that? REM Microsoft Visual Studio 2010 (VC=10.0) REM Microsoft Visual Studio 2012 (VC=11.0) REM Microsoft Visual Studio 2013 (VC=12.0) -REM Microsoft Visual Studio 2015 (VC=14.0) +REM Microsoft Visual Studio 2015 (VC=14.0) -> for: x32 + x64 REM Microsoft Visual Studio 2017 (VC=15.0) REM to many vcvarsall.cmd calls will blow it up! @@ -12,50 +12,52 @@ set OPATH=%PATH% set ERRFILE=%APPVEYOR_BUILD_FOLDER%\error.txt cd %APPVEYOR_BUILD_FOLDER%\CPP -REM I am using sdk71 and 12.0 for releases... /TR +REM I am using VC 14.0 for releases now... /TR 2018-11-15 +goto vc14 :sdk71 set VC=sdk71 -set NEXT=vc12 +set NEXT=vc14 goto build_sdk :vc11 set VC=11.0 set NEXT=vc12 -goto build +set CFLAGS=-Gw +goto build_vc :vc12 set VC=12.0 -set CFLAGS=-Gw set NEXT=end -goto build +set CFLAGS=-Gw +goto build_vc :vc14 set VC=14.0 set NEXT=end -goto build +set CFLAGS=-Gw +goto build_vc -:build -FOR /R .\ %%d IN (AMD64 O) DO rd /S /Q %%d 2>NUL +:build_vc +FOR /R .\ %%d IN (ARM X64 O) DO rd /S /Q %%d 2>NUL set PATH=%OPATH% call "C:\Program Files (x86)\Microsoft Visual Studio %VC%\VC\vcvarsall.bat" x86 set OUTDIR=%APPVEYOR_BUILD_FOLDER%\bin-%VC%-x32 call build-x32.cmd +set PATH=%OPATH% call "C:\Program Files (x86)\Microsoft Visual Studio %VC%\VC\vcvarsall.bat" x86_amd64 set OUTDIR=%APPVEYOR_BUILD_FOLDER%\bin-%VC%-x64 call build-x64.cmd +set PATH=%OPATH% +call "C:\Program Files (x86)\Microsoft Visual Studio %VC%\VC\vcvarsall.bat" x86_arm +set OUTDIR=%APPVEYOR_BUILD_FOLDER%\bin-%VC%-arm +call build-arm.cmd goto %NEXT% :build_sdk set PATH=%OPATH% call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" x86 -REM call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /x86 /xp -REM set OUTDIR=%APPVEYOR_BUILD_FOLDER%\bin-%VC%-x32 -REM call build-x32.cmd -REM call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /x64 /xp -REM set OUTDIR=%APPVEYOR_BUILD_FOLDER%\bin-%VC%-x64 -REM call build-x64.cmd call "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /Release /ia64 /xp set OUTDIR=%APPVEYOR_BUILD_FOLDER%\bin-%VC%-ia64 call build-ia64.cmd diff --git a/CPP/build-arm.cmd b/CPP/build-arm.cmd new file mode 100644 index 00000000..90cac9ff --- /dev/null +++ b/CPP/build-arm.cmd @@ -0,0 +1,101 @@ +@echo off + +set ROOT=%cd%\7zip +if not defined OUTDIR set OUTDIR=%ROOT%\binArm +if not defined ERRFILE set ERRFILE=%cd%\error.txt +mkdir %OUTDIR% + +set OPTS=PLATFORM=arm MY_STATIC_LINK=1 /NOLOGO + +cd %ROOT%\Bundles\Format7zExtract +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7zxa.dll" >> %ERRFILE% +copy arm\7zxa.dll %OUTDIR%\7zxa.dll + +cd %ROOT%\Bundles\Format7z +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7za.dll" >> %ERRFILE% +copy arm\7za.dll %OUTDIR%\7za.dll + +cd %ROOT%\Bundles\Format7zF +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7z.dll" >> %ERRFILE% +copy arm\7z.dll %OUTDIR%\7z.dll + +REM cd %ROOT%\UI\FileManager +REM nmake %OPTS% +REM IF %errorlevel% NEQ 0 echo "Error x64 @ 7zFM.exe" >> %ERRFILE% +REM copy arm\7zFM.exe %OUTDIR%\7zFM.exe + +REM cd %ROOT%\UI\GUI +REM nmake %OPTS% +REM IF %errorlevel% NEQ 0 echo "Error x64 @ 7zG.exe" >> %ERRFILE% +REM copy arm\7zG.exe %OUTDIR%\7zG.exe + +cd %ROOT%\UI\Explorer +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7-zip.dll" >> %ERRFILE% +copy arm\7-zip.dll %OUTDIR%\7-zip.dll + +REM cd %ROOT%\Bundles\SFXWin +REM nmake %OPTS% +REM IF %errorlevel% NEQ 0 echo "Error x64 @ 7z.sfx" >> %ERRFILE% +REM copy arm\7z.sfx %OUTDIR%\7z.sfx + +cd %ROOT%\Bundles\Codec_brotli +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ brotli-arm.dll" >> %ERRFILE% +copy arm\brotli.dll %OUTDIR%\brotli-arm.dll + +cd %ROOT%\Bundles\Codec_lizard +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ lizard-arm.dll" >> %ERRFILE% +copy arm\lizard.dll %OUTDIR%\lizard-arm.dll + +cd %ROOT%\Bundles\Codec_lz4 +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ lz4-arm.dll" >> %ERRFILE% +copy arm\lz4.dll %OUTDIR%\lz4-arm.dll + +cd %ROOT%\Bundles\Codec_lz5 +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ lz5-arm.dll" >> %ERRFILE% +copy arm\lz5.dll %OUTDIR%\lz5-arm.dll + +cd %ROOT%\Bundles\Codec_zstd +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ zstd-arm.dll" >> %ERRFILE% +copy arm\zstd.dll %OUTDIR%\zstd-arm.dll + +cd %ROOT%\Bundles\Codec_flzma2 +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ flzma2-arm.dll" >> %ERRFILE% +copy arm\flzma2.dll %OUTDIR%\flzma2-arm.dll + +REM cd %ROOT%\..\..\C\Util\7zipInstall +REM nmake %OPTS% +REM IF %errorlevel% NEQ 0 echo "Error x64 @ Install-arm.exe" >> %ERRFILE% +REM copy arm\7zipInstall.exe %OUTDIR%\Install-arm.exe + +REM cd %ROOT%\..\..\C\Util\7zipUninstall +REM nmake %OPTS% +REM IF %errorlevel% NEQ 0 echo "Error x64 @ Uninstall.exe" >> %ERRFILE% +REM copy arm\7zipUninstall.exe %OUTDIR%\Uninstall.exe + +cd %ROOT%\UI\Console +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7z.exe" >> %ERRFILE% +copy arm\7z.exe %OUTDIR%\7z.exe + +cd %ROOT%\Bundles\SFXCon +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7zCon.sfx" >> %ERRFILE% +copy arm\7zCon.sfx %OUTDIR%\7zCon.sfx + +cd %ROOT%\Bundles\Alone +nmake %OPTS% +IF %errorlevel% NEQ 0 echo "Error x64 @ 7za.exe" >> %ERRFILE% +copy arm\7za.exe %OUTDIR%\7za.exe + +:ende +cd %ROOT%\.. diff --git a/CPP/build-ia64.cmd b/CPP/build-ia64.cmd deleted file mode 100644 index 09d89c6e..00000000 --- a/CPP/build-ia64.cmd +++ /dev/null @@ -1,112 +0,0 @@ -@echo off - -set ROOT=%cd%\7zip -if not defined OUTDIR set OUTDIR=%ROOT%\binIA64 -mkdir %OUTDIR% - -set OPTS=CPU=IA64 MY_STATIC_LINK=1 /NOLOGO /TP -set LFLAGS=/SUBSYSTEM:WINDOWS,"5.02" - -cd %ROOT%\Bundles\Format7zExtract -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7zxa.dll" >> %ERRFILE% -copy IA64\7zxa.dll %OUTDIR%\7zxa.dll - -cd %ROOT%\Bundles\Format7z -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7za.dll" >> %ERRFILE% -copy IA64\7za.dll %OUTDIR%\7za.dll - -cd %ROOT%\Bundles\Format7zF -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7z.dll" >> %ERRFILE% -copy IA64\7z.dll %OUTDIR%\7z.dll - -cd %ROOT%\Bundles\Format7zFO -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7zOrig.dll" >> %ERRFILE% -copy IA64\7z.dll %OUTDIR%\7zOrig.dll - -cd %ROOT%\Bundles\Format7zUSB -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7zUSB.dll" >> %ERRFILE% -copy IA64\7zu.dll %OUTDIR%\7zu-x64.dll - -cd %ROOT%\UI\FileManager -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7zFM.exe" >> %ERRFILE% -copy IA64\7zFM.exe %OUTDIR%\7zFM.exe - -cd %ROOT%\UI\GUI -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7zG.exe" >> %ERRFILE% -copy IA64\7zG.exe %OUTDIR%\7zG.exe - -cd %ROOT%\UI\Explorer -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7-zip.dll" >> %ERRFILE% -copy IA64\7-zip.dll %OUTDIR%\7-zip.dll - -cd %ROOT%\Bundles\SFXWin -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7z.sfx" >> %ERRFILE% -copy IA64\7z.sfx %OUTDIR%\7z.sfx - -cd %ROOT%\Bundles\Codec_brotli -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ brotli-x64.dll" >> %ERRFILE% -copy IA64\brotli.dll %OUTDIR%\brotli-x64.dll - -cd %ROOT%\Bundles\Codec_lizard -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ lizard-x64.dll" >> %ERRFILE% -copy IA64\lizard.dll %OUTDIR%\lizard-x64.dll - -cd %ROOT%\Bundles\Codec_lz4 -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ lz4-x64.dll" >> %ERRFILE% -copy IA64\lz4.dll %OUTDIR%\lz4-x64.dll - -cd %ROOT%\Bundles\Codec_lz5 -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ lz5-x64.dll" >> %ERRFILE% -copy IA64\lz5.dll %OUTDIR%\lz5-x64.dll - -cd %ROOT%\Bundles\Codec_zstdF -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ zstd-x64.dll" >> %ERRFILE% -copy IA64\zstd.dll %OUTDIR%\zstd-x64.dll - -cd %ROOT%\Bundles\Codec_flzma2 -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ flzma2-x64.dll" >> %ERRFILE% -copy IA64\flzma2.dll %OUTDIR%\flzma2-x64.dll - -cd %ROOT%\..\..\C\Util\7zipInstall -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ Install-x64.exe" >> %ERRFILE% -copy IA64\7zipInstall.exe %OUTDIR%\Install-x64.exe - -cd %ROOT%\..\..\C\Util\7zipUninstall -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ Uninstall.exe" >> %ERRFILE% -copy IA64\7zipUninstall.exe %OUTDIR%\Uninstall.exe - -set LFLAGS=/SUBSYSTEM:CONSOLE,"5.02" -cd %ROOT%\UI\Console -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7z.exe" >> %ERRFILE% -copy IA64\7z.exe %OUTDIR%\7z.exe - -cd %ROOT%\Bundles\SFXCon -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7zCon.sfx" >> %ERRFILE% -copy IA64\7zCon.sfx %OUTDIR%\7zCon.sfx - -cd %ROOT%\Bundles\Alone -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error ia64 @ 7za.exe" >> %ERRFILE% -copy IA64\7za.exe %OUTDIR%\7za.exe - -:ende -cd %ROOT%\.. diff --git a/CPP/build-x32.cmd b/CPP/build-x32.cmd index d10c3f47..d93e6893 100644 --- a/CPP/build-x32.cmd +++ b/CPP/build-x32.cmd @@ -23,16 +23,6 @@ nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x32 @ 7z.dll" >> %ERRFILE% copy O\7z.dll %OUTDIR%\7z.dll -cd %ROOT%\Bundles\Format7zFO -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error x32 @ 7zOrig.dll" >> %ERRFILE% -copy O\7z.dll %OUTDIR%\7zOrig.dll - -cd %ROOT%\Bundles\Format7zUSB -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error x32 @ 7zUSB.dll" >> %ERRFILE% -copy O\7zu.dll %OUTDIR%\7zu-x32.dll - cd %ROOT%\UI\FileManager nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x32 @ 7zFM.exe" >> %ERRFILE% @@ -73,7 +63,7 @@ nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x32 @ lz5-x32.dll" >> %ERRFILE% copy O\lz5.dll %OUTDIR%\lz5-x32.dll -cd %ROOT%\Bundles\Codec_zstdF +cd %ROOT%\Bundles\Codec_zstd nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x32 @ zstd-x32.dll" >> %ERRFILE% copy O\zstd.dll %OUTDIR%\zstd-x32.dll diff --git a/CPP/build-x64.cmd b/CPP/build-x64.cmd index a649fa1f..1b38a9f7 100644 --- a/CPP/build-x64.cmd +++ b/CPP/build-x64.cmd @@ -5,109 +5,99 @@ if not defined OUTDIR set OUTDIR=%ROOT%\bin64 if not defined ERRFILE set ERRFILE=%cd%\error.txt mkdir %OUTDIR% -set OPTS=CPU=AMD64 MY_STATIC_LINK=1 /NOLOGO +set OPTS=PLATFORM=x64 MY_STATIC_LINK=1 /NOLOGO set LFLAGS=/SUBSYSTEM:WINDOWS,"5.02" cd %ROOT%\Bundles\Format7zExtract nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7zxa.dll" >> %ERRFILE% -copy AMD64\7zxa.dll %OUTDIR%\7zxa.dll +copy X64\7zxa.dll %OUTDIR%\7zxa.dll cd %ROOT%\Bundles\Format7z nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7za.dll" >> %ERRFILE% -copy AMD64\7za.dll %OUTDIR%\7za.dll +copy X64\7za.dll %OUTDIR%\7za.dll cd %ROOT%\Bundles\Format7zF nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7z.dll" >> %ERRFILE% -copy AMD64\7z.dll %OUTDIR%\7z.dll - -cd %ROOT%\Bundles\Format7zFO -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error x64 @ 7zOrig.dll" >> %ERRFILE% -copy AMD64\7z.dll %OUTDIR%\7zOrig.dll - -cd %ROOT%\Bundles\Format7zUSB -nmake %OPTS% -IF %errorlevel% NEQ 0 echo "Error x64 @ 7zUSB.dll" >> %ERRFILE% -copy AMD64\7zu.dll %OUTDIR%\7zu-x64.dll +copy X64\7z.dll %OUTDIR%\7z.dll cd %ROOT%\UI\FileManager nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7zFM.exe" >> %ERRFILE% -copy AMD64\7zFM.exe %OUTDIR%\7zFM.exe +copy X64\7zFM.exe %OUTDIR%\7zFM.exe cd %ROOT%\UI\GUI nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7zG.exe" >> %ERRFILE% -copy AMD64\7zG.exe %OUTDIR%\7zG.exe +copy X64\7zG.exe %OUTDIR%\7zG.exe cd %ROOT%\UI\Explorer nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7-zip.dll" >> %ERRFILE% -copy AMD64\7-zip.dll %OUTDIR%\7-zip.dll +copy X64\7-zip.dll %OUTDIR%\7-zip.dll cd %ROOT%\Bundles\SFXWin nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7z.sfx" >> %ERRFILE% -copy AMD64\7z.sfx %OUTDIR%\7z.sfx +copy X64\7z.sfx %OUTDIR%\7z.sfx cd %ROOT%\Bundles\Codec_brotli nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ brotli-x64.dll" >> %ERRFILE% -copy AMD64\brotli.dll %OUTDIR%\brotli-x64.dll +copy X64\brotli.dll %OUTDIR%\brotli-x64.dll cd %ROOT%\Bundles\Codec_lizard nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ lizard-x64.dll" >> %ERRFILE% -copy AMD64\lizard.dll %OUTDIR%\lizard-x64.dll +copy X64\lizard.dll %OUTDIR%\lizard-x64.dll cd %ROOT%\Bundles\Codec_lz4 nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ lz4-x64.dll" >> %ERRFILE% -copy AMD64\lz4.dll %OUTDIR%\lz4-x64.dll +copy X64\lz4.dll %OUTDIR%\lz4-x64.dll cd %ROOT%\Bundles\Codec_lz5 nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ lz5-x64.dll" >> %ERRFILE% -copy AMD64\lz5.dll %OUTDIR%\lz5-x64.dll +copy X64\lz5.dll %OUTDIR%\lz5-x64.dll -cd %ROOT%\Bundles\Codec_zstdF +cd %ROOT%\Bundles\Codec_zstd nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ zstd-x64.dll" >> %ERRFILE% -copy AMD64\zstd.dll %OUTDIR%\zstd-x64.dll +copy X64\zstd.dll %OUTDIR%\zstd-x64.dll cd %ROOT%\Bundles\Codec_flzma2 nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ flzma2-x64.dll" >> %ERRFILE% -copy AMD64\flzma2.dll %OUTDIR%\flzma2-x64.dll +copy X64\flzma2.dll %OUTDIR%\flzma2-x64.dll cd %ROOT%\..\..\C\Util\7zipInstall nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ Install-x64.exe" >> %ERRFILE% -copy AMD64\7zipInstall.exe %OUTDIR%\Install-x64.exe +copy X64\7zipInstall.exe %OUTDIR%\Install-x64.exe cd %ROOT%\..\..\C\Util\7zipUninstall nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ Uninstall.exe" >> %ERRFILE% -copy AMD64\7zipUninstall.exe %OUTDIR%\Uninstall.exe +copy X64\7zipUninstall.exe %OUTDIR%\Uninstall.exe set LFLAGS=/SUBSYSTEM:CONSOLE,"5.02" cd %ROOT%\UI\Console nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7z.exe" >> %ERRFILE% -copy AMD64\7z.exe %OUTDIR%\7z.exe +copy X64\7z.exe %OUTDIR%\7z.exe cd %ROOT%\Bundles\SFXCon nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7zCon.sfx" >> %ERRFILE% -copy AMD64\7zCon.sfx %OUTDIR%\7zCon.sfx +copy X64\7zCon.sfx %OUTDIR%\7zCon.sfx cd %ROOT%\Bundles\Alone nmake %OPTS% IF %errorlevel% NEQ 0 echo "Error x64 @ 7za.exe" >> %ERRFILE% -copy AMD64\7za.exe %OUTDIR%\7za.exe +copy X64\7za.exe %OUTDIR%\7za.exe :ende cd %ROOT%\.. diff --git a/DOC/7zFormat.txt b/DOC/7zFormat.txt index 6b8678fb..9239e935 100644 --- a/DOC/7zFormat.txt +++ b/DOC/7zFormat.txt @@ -1,4 +1,4 @@ -7z Format description (4.59) +7z Format description (18.06) ---------------------------- This file contains description of 7z archive format. @@ -175,7 +175,7 @@ SignatureHeader ArchiveVersion { BYTE Major; // now = 0 - BYTE Minor; // now = 2 + BYTE Minor; // now = 4 }; UINT32 StartHeaderCRC; @@ -399,7 +399,7 @@ FilesInfo UINT64 DataIndex [] for(Definded Items) - UINT64 Time + REAL_UINT64 Time [] kNames: (0x11) diff --git a/DOC/7zip.inf b/DOC/7zip.inf index 4f79b44c..ce07d931 100644 --- a/DOC/7zip.inf +++ b/DOC/7zip.inf @@ -10,8 +10,8 @@ AppName = "7-Zip" InstallDir = %CE1%\%AppName% [Strings] -AppVer = "18.05" -AppDate = "2018-04-30" +AppVer = "19.00" +AppDate = "2019-01-21" [CEDevice] ; ProcessorType = 2577 ; ARM diff --git a/DOC/7zip.nsi b/DOC/7zip.nsi index ae55c49c..fef7403d 100644 --- a/DOC/7zip.nsi +++ b/DOC/7zip.nsi @@ -1,8 +1,8 @@ ;-------------------------------- ;Defines -!define VERSION_MAJOR 18 -!define VERSION_MINOR 05 +!define VERSION_MAJOR 19 +!define VERSION_MINOR 00 !define VERSION_POSTFIX_FULL " ZS" !ifdef WIN64 !ifdef IA64 @@ -30,7 +30,7 @@ !define FM_LINK "7-Zip File Manager.lnk" !define HELP_LINK "7-Zip Help.lnk" -!define CLSID_CONTEXT_MENU {23170F69-0803-278A-1000-000100020000} +!define CLSID_CONTEXT_MENU {23170F69-40C1-278A-1000-000100020000} #!define NO_COMPRESSION @@ -536,7 +536,6 @@ Section Uninstall DeleteRegKey HKCR 7-Zip.hfs DeleteRegKey HKCR 7-Zip.iso DeleteRegKey HKCR 7-Zip.lha - DeleteRegKey HKCR 7-Zip.lz DeleteRegKey HKCR 7-Zip.lzh DeleteRegKey HKCR 7-Zip.lzma DeleteRegKey HKCR 7-Zip.ntfs @@ -548,7 +547,6 @@ Section Uninstall DeleteRegKey HKCR 7-Zip.tar DeleteRegKey HKCR 7-Zip.taz DeleteRegKey HKCR 7-Zip.tgz - DeleteRegKey HKCR 7-Zip.tlz DeleteRegKey HKCR 7-Zip.tpz DeleteRegKey HKCR 7-Zip.txz DeleteRegKey HKCR 7-Zip.vhd diff --git a/DOC/7zip.wxs b/DOC/7zip.wxs index f97502d5..93f05f96 100644 --- a/DOC/7zip.wxs +++ b/DOC/7zip.wxs @@ -1,7 +1,7 @@ - - + + @@ -38,9 +38,9 @@ - + - + diff --git a/DOC/License.txt b/DOC/License.txt index cae06938..e9113842 100644 --- a/DOC/License.txt +++ b/DOC/License.txt @@ -3,7 +3,7 @@ License for use and distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 7-Zip Copyright (C) 1999-2018 Igor Pavlov. + 7-Zip Copyright (C) 1999-2019 Igor Pavlov. The licenses for files are: diff --git a/DOC/Methods-Extern.md b/DOC/Methods-Extern.md index e252dee8..17702cf5 100644 --- a/DOC/Methods-Extern.md +++ b/DOC/Methods-Extern.md @@ -62,22 +62,22 @@ License: Zstandard library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 5 bytes. The data within that header +This header is mandatory and must be exact 3 or 5 bytes. The data within that header is for informational purposes only and not used by the decoder. If the header is not there, or has another size, the decoder will not decompress the content. ``` C - Byte _ver_major; // currently 1 - Byte _ver_minor; // currently 2 - Byte _level; // currently 1..22 - Byte _reserved[2]; + Byte _ver_major; // currently 1 + Byte _ver_minor; // currently 2 + Byte _level; // currently 1..22 or 33..MaxFastLevel + Byte _reserved[2]; // not given in 3 byte header ``` - this header holds some information about the version, which was used for creating that 7-Zip container data - _ver_major should contain the major release of zstd - _ver_minor should contain the major release of zstd - _level should contain the level, the data is packed with -- the other two bytes should be set to zero currently and are - reserved for future use +- negative fast level values are stored as fastlevel+32 (33 = fastlevel 1 and so on..) +- the other two bytes, if present, should be set to zero currently and are reserved for future use Algorithm author: Yann Collet - Homepage: https://facebook.github.io/zstd/ @@ -164,22 +164,21 @@ License: LZ4 library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 5 bytes. The data within that header +This header is mandatory and must be exact 3 or 5 bytes. The data within that header is for informational purposes only and not used by the decoder. If the header is not there, or has another size, the decoder will not decompress the content. ``` C - Byte _ver_major; // currently 1 - Byte _ver_minor; // currently 7 - Byte _level; // 1..12 - Byte _reserved[2]; + Byte _ver_major; // currently 1 + Byte _ver_minor; // currently 7 + Byte _level; // 1..12 + Byte _reserved[2]; // not given in 3 byte header ``` - this header holds some information about the version, which was used for creating that 7-Zip container data - _ver_major should contain the major release of LZ4 - _ver_minor should contain the major release of LZ4 - _level should contain the level, the data is packed with -- the other two bytes should be set to zero currently and are - reserved for future use +- the other two bytes, if present, should be set to zero currently and are reserved for future use Algorithm author: Yann Collet - Homepage: https://lz4.github.io/lz4/ @@ -215,22 +214,21 @@ License: LZ5 library is provided as open source software using the BSD license. 7-Zip Container Header: -This header is mandatory and must be exact 5 bytes. The data within that header +This header is mandatory and must be exact 3 or 5 bytes. The data within that header is for informational purposes only and not used by the decoder. If the header is not there, or has another size, the decoder will not decompress the content. ``` C - Byte _ver_major; // currently 1 - Byte _ver_minor; // currently 5 - Byte _level; // 1..15 - Byte _reserved[2]; + Byte _ver_major; // currently 1 + Byte _ver_minor; // currently 5 + Byte _level; // 1..15 + Byte _reserved[2]; // not given in 3 byte header ``` - this header holds some information about the version, which was used for creating that 7-Zip container data - _ver_major should contain the major release of LZ5 - _ver_minor should contain the major release of LZ5 - _level should contain the level, the data is packed with -- the other two bytes should be set to zero currently and are reserved for - future use +- the other two bytes, if present, should be set to zero currently and are reserved for future use Algorithm author: Przemyslaw Skibinski - Homepage: https://github.com/inikep/lz5 diff --git a/DOC/Methods.txt b/DOC/Methods.txt index 11adcb0b..6d0641ba 100644 --- a/DOC/Methods.txt +++ b/DOC/Methods.txt @@ -1,8 +1,8 @@ 7-Zip method IDs for 7z and xz archives --------------------------------------- -Version: 17.01 -Date: 2017-05-27 +Version: 18.06 +Date: 2018-06-30 Each compression or crypto method in 7z is associated with unique binary value (ID). The length of ID in bytes is arbitrary but it can not exceed 63 bits (8 bytes). @@ -132,6 +132,12 @@ List of defined IDs 04 - LZ4 05 - LZ5 06 - LIZARD + + 12 xx - reserverd (Denis Anisimov) + + 01 - WavPack2 + FE - eSplitter + FF - RawSplitter 06.. - Crypto diff --git a/DOC/readme.txt b/DOC/readme.txt index 3118bdeb..9dd6e084 100644 --- a/DOC/readme.txt +++ b/DOC/readme.txt @@ -1,9 +1,9 @@ -7-Zip 18.05 Sources +7-Zip 19.00 Sources ------------------- 7-Zip is a file archiver for Windows. -7-Zip Copyright (C) 1999-2018 Igor Pavlov. +7-Zip Copyright (C) 1999-2019 Igor Pavlov. License Info @@ -44,28 +44,50 @@ LZMA SDK is written and placed in the public domain by Igor Pavlov. How to compile -------------- -To compile sources you need Visual C++ 6.0. -For compiling some files you also need -new Platform SDK from Microsoft' Site: -http://www.microsoft.com/msdownload/platformsdk/sdkupdate/psdk-full.htm -or -http://www.microsoft.com/msdownload/platformsdk/sdkupdate/XPSP2FULLInstall.htm -or -http://www.microsoft.com/msdownload/platformsdk/sdkupdate/ -If you use MSVC6, specify SDK directories at top of directories lists: +To compile the sources to Windows binaries you need Visual Studio compiler and/or Windows SDK. +You can use latest Windows Studio 2017 to compile binaries for x86, x64 and arm64 platforms. +Also you can use old compilers for some platforms: + x86 : Visual C++ 6.0 with Platform SDK + x64 : Windows Server 2003 R2 Platform SDK + arm64 : Windows Studio 2017 + arm : Windows Studio 2017 + ia64 (itanium) : Windows Server 2003 R2 Platform SDK + arm for Windows CE : Standard SDK for Windows CE 5.0 + +If you use MSVC6, specify also Platform SDK directories at top of directories lists: Tools / Options / Directories - Include files - Library files - -To compile 7-Zip for AMD64 and IA64 you need: - Windows Server 2003 SP1 Platform SDK from microsoft.com - Also you need Microsoft Macro Assembler: - ml.exe for x86 - - ml64.exe for AMD64 -You can use ml.exe from Windows SDK for Windows Vista or some other version. + - ml64.exe for x64 +You can use ml.exe from Windows SDK for Windows Vista or some later versions. + +There are two ways to compile 7-Zip binaries: +1) via makefile in command line. +2) via dsp file in Visual Studio. + +The dsp file compiling can be used for development and debug purposes. +The final 7-Zip binaries are compiled via makefiles, that provide best +optimization options. + +How to compile with makefile +---------------------------- + +Some macronames can be defined for compiling with makefile: + +PLATFORM + with possible values: x64, x86, arm64, arm, ia64 + +OLD_COMPILER + for old VC compiler, like MSCV 6.0. + +MY_DYNAMIC_LINK + for dynamic linking to the run-time library (msvcrt.dll). + The default makefile option is static linking to the run-time library. + Compiling under Unix/Linux @@ -123,7 +145,7 @@ Windows common files for Windows related code Bundle Modules that are bundles of other modules (files) - Alone 7za.exe: Standalone version of 7-Zip console that supports only 7z/xz/cab/zip/gzip/bzip2/lzip/tar + Alone 7za.exe: Standalone version of 7-Zip console that supports only 7z/xz/cab/zip/gzip/bzip2/tar. Alone7z 7zr.exe: Standalone version of 7-Zip console that supports only 7z (reduced version) Fm Standalone version of 7-Zip File Manager Format7z 7za.dll: .7z support diff --git a/DOC/src-history.txt b/DOC/src-history.txt index 57db3785..7fcb09a0 100644 --- a/DOC/src-history.txt +++ b/DOC/src-history.txt @@ -1,6 +1,29 @@ HISTORY of the 7-Zip source code -------------------------------- +19.00 2019-02-21 +------------------------- +- Encryption strength for 7z archives was increased: + the size of random initialization vector was increased from 64-bit to 128-bit, + and the pseudo-random number generator was improved. +- Some bugs were fixed. + + +18.06 2018-12-30 +------------------------- +- The speed for LZMA/LZMA2 compressing was increased by 3-10%, + and there are minor changes in compression ratio. +- Some bugs were fixed. +- The bug in 7-Zip 18.02-18.05 was fixed: + There was memory leak in multithreading xz decoder - XzDecMt_Decode(), + if xz stream contains only one block. +- 7-Zip 18.02-18.05 used only one CPU thread for bz2 archive creation. +- The changes for MSVS compiler makefiles: + - the makefiles now use "PLATFORM" macroname with values (x64, x86, arm64) + instead of "CPU" macroname with values (AMD64, ARM64). + - the makefiles by default now use static version of the run-time library. + + 18.05 2018-04-30 ------------------------- - The speed for LZMA/LZMA2 compressing was increased diff --git a/README.md b/README.md index 04aabe6a..a1ef0e7b 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ This is the Github Page of [7-Zip] ZS with support of additional Codecs. The lib You can install it in two ways: 1. complete setup with additions within the GUI and a modified Explorer context menu -2. only the codec plugin that goes to your existing [7-Zip] installation +2. only the codec plugin that goes to your existing [7-Zip] installation (no GUI changes and no additional Hashers) # Status @@ -15,10 +15,10 @@ You can install it in two ways: ## Codec overview -1. [Zstandard] v1.3.7 is a real-time compression algorithm, providing high compression ratios. It offers a very wide range of compression / speed trade-off, while being backed by a very fast decoder. +1. [Zstandard] v1.3.8 is a real-time compression algorithm, providing high compression ratios. It offers a very wide range of compression / speed trade-off, while being backed by a very fast decoder. - Levels: 1..22 -2. [Brotli] v.1.0.6 is a generic-purpose lossless compression algorithm that compresses data using a combination of a modern variant of the LZ77 algorithm, Huffman coding and 2nd order context modeling, with a compression ratio comparable to the best currently available general-purpose compression methods. It is similar in speed with deflate but offers more dense compression. +2. [Brotli] v.1.0.7 is a generic-purpose lossless compression algorithm that compresses data using a combination of a modern variant of the LZ77 algorithm, Huffman coding and 2nd order context modeling, with a compression ratio comparable to the best currently available general-purpose compression methods. It is similar in speed with deflate but offers more dense compression. - Levels: 0..11 3. [LZ4] v1.8.3 is lossless compression algorithm, providing compression speed at 400 MB/s per core (0.16 Bytes/cycle). It features an extremely fast decoder, with speed in multiple GB/s per core (0.71 Bytes/cycle). A high compression derivative, called LZ4_HC, is available, trading customizable CPU time for compression ratio. @@ -43,16 +43,15 @@ You can install it in two ways: 2. install it, like the default [7-Zip] 3. use it ;) 4. you may check, if the [7-Zip] can deal with [Zstandard] or other codecs via this command: `7z.exe i` -5. the binaries within this installation are not binary compatible with [7-Zip]... use therefore the files from the `Codecs.7z` archive The output should look like this: ``` -7-Zip 18.05 ZS v1.3.7 R1 (x64) : Copyright (c) 1999-2017 Igor Pavlov : 2018-10-21 - +7-Zip 19.00 ZS v1.3.8 R2 (x64) : Copyright (c) 1999-2019 Igor Pavlov, 2016-2019 Tino Reichardt : 2019-03-05 Libs: 0 c:\Program Files\7-Zip-Zstandard\7z.dll - + 1 C:\Program Files\7-Zip-Zstandard\Codecs\Iso7z.64.dll + Formats: ... 0 CK xz xz txz (.tar) FD 7 z X Z 00 @@ -92,13 +91,28 @@ Codecs: 0 ED 21 FLZMA2 0 ED 6F10701 7zAES 0 ED 6F00181 AES256CBC + 1 3ED 4F712FF RawSplitter + +Hashers: + 0 4 1 CRC32 + 0 16 205 MD2 + 0 16 206 MD4 + 0 16 207 MD5 + 0 20 201 SHA1 + 0 32 A SHA256 + 0 48 208 SHA384 + 0 64 209 SHA512 + 0 4 203 XXH32 + 0 8 204 XXH64 + 0 8 4 CRC64 + 0 32 202 BLAKE2sp ``` ### Usage and features of the full installation - compression and decompression for [Brotli], [Lizard], [LZ4], [LZ5] and [Zstandard] within the [7-Zip] container format - compression and decompression of [Lizard] (`.liz`), [LZ4] (`.lz4`), [LZ5] (`.lz5`) and [Zstandard] (`.zst`) files -- included [lzip] decompression support, patch from: http://download.savannah.gnu.org/releases/lzip/7zip/ +- included [lzip] decompression support, patch from: https://download.savannah.gnu.org/releases/lzip/7zip/ - right click and _"Add to xy.7z"_ will use the last selected method (codec, level and threads) - the FileManager ListBox will show more information about these codecs now @@ -132,7 +146,8 @@ Codecs: ![Explorer inegration](https://mcmilk.de/projects/7-Zip-zstd/Add-To-Archive.png "Add to Archiv Dialog with ZSTD options") ![File Manager](https://mcmilk.de/projects/7-Zip-zstd/Fileman.png "File Manager with the Listing of an Archiv") -![Methods](https://mcmilk.de/projects/7-Zip-zstd/Methods.png "Methods") +![Methods](https://mcmilk.de/projects/7-Zip-zstd/Methods2.png "Methods") +![Hashes](https://mcmilk.de/projects/7-Zip-zstd/Hashes.png "Hashes") ## Zstandard codec Plugin for Mainline 7-Zip @@ -147,7 +162,7 @@ Codecs: The output should look like this: ``` -7-Zip 18.05 (x64) : Copyright (c) 1999-2018 Igor Pavlov : 2018-04-30 +7-Zip 19.00 (x64) : Copyright (c) 1999-2019 Igor Pavlov : 2018-04-30 Libs: 0 C:\Program Files\7-Zip\7z.dll @@ -197,7 +212,7 @@ Codecs: ### Usage (codec plugin) -- compression and decompression for [Brotli], [Lizard], [LZ4], [LZ5] and [Zstandard] within the 7-Zip container format +- compression and decompression for [Brotli], [Fast LZMA2], [Lizard], [LZ4], [LZ5] and [Zstandard] within the 7-Zip container format - you can only create `.7z` files, the files like `.lz4`, `.lz5` and `.zst` are not covered by the plugins - when compressing binaries (*.exe, *.dll), you have to explicitly disable the bcj2 filter via `-m0=bcj`, when using only the plugin dll's @@ -217,14 +232,12 @@ Codecs: ## Codec Plugin for Total Commander - download [TotalCmd.7z] - install it, by replacing the files `tc7z.dll` and `tc7z64.dll` with the new ones -- you can check the Total Commander Forum for more information about this [DLL Files](http://ghisler.ch/board/viewtopic.php?p=319216) +- you can check the Total Commander Forum for more information about this [DLL Files](https://ghisler.ch/board/viewtopic.php?p=319216) - decompression for [Brotli], [Lizard], [LZ4], [LZ5] and [Zstandard] ot the 7-Zip `.7z` format will work out of the box with Total Commander now :-) ## Codec Plugin for Far Manager -- download [Codecs.7z] -- install it, by replacing the file `C:\Program Files\Far Manager\Plugins\ArcLite\7z.dll` with the one found in the [Codecs.7z] archive -- it's named `7z-x64.dll` or `7z-x32.dll`, depending on your architecture +- copy the `7z.dll` file from `C:\Program Files\7-Zip-Zstandard\7z.dll` to `C:\Program Files\Far Manager\Plugins\ArcLite\7z.dll` - then restart the Far manager - and on next start, you will have support for 7-Zip Zstandard archives ;-) ## Benchmarks @@ -235,6 +248,8 @@ For the benchmarks I am using Windows 7 64bit on my Laptop which has the followi - the decompression benchmark is also done in RAM via: `7z t archiv.7z` - the tool for measuring the times is [wtime](https://github.com/mcmilk/wtime), together with some [scripts](https://github.com/mcmilk/7-Zip-Benchmarking) - the testfile is generated via [generate-mcorpus](https://github.com/mcmilk/7-Zip-Benchmarking/blob/master/generate-mcorpus) +- please note, that the benchmarks are outdated now (new one will come, but it needs some time) + ![Compression Speed vs Ratio](https://mcmilk.de/projects/7-Zip-zstd/dl/compr-v120.png "Compression Speed vs Ratio") ![Decompression Speed](https://mcmilk.de/projects/7-Zip-zstd/dl/decomp-v120.png "Decompression Speed per Level") ![Memory at Compression](https://mcmilk.de/projects/7-Zip-zstd/dl/MemCompr.png "Memory usage at Compression") @@ -246,7 +261,7 @@ For the benchmarks I am using Windows 7 64bit on my Laptop which has the followi ## Links -- [7-Zip Homepage](http://www.7-zip.org/) +- [7-Zip Homepage](https://www.7-zip.org/) - [7-Zip Zstandard Homepage](https://mcmilk.de/projects/7-Zip-zstd/) - [Request for inclusion](https://sourceforge.net/p/sevenzip/discussion/45797/thread/a7e4f3f3/) into the mainline 7-Zip: - result, will currently not included :( @@ -259,23 +274,22 @@ You find this project useful, maybe you consider a donation ;-) ## Version Information -- 7-Zip ZS Version 18.05 - - [Brotli] Version 1.0.6 +- 7-Zip ZS Version 19.00 + - [Brotli] Version 1.0.7 + - [Fast LZMA2] Version 0.9.2 - [Lizard] Version 1.0 - [LZ4] Version 1.8.3 - [LZ5] Version 1.5 - - [Zstandard] Version 1.3.7 - - [Fast LZMA2] Version 0.9.2 + - [Zstandard] Version 1.3.8 -/TR 2018-10-21 +/TR 2019-03-05 ## Notes - if you want an code signed installer, you need to donate sth. -- I know that Zstandard has some nice new features which are not included currently... I will think about it -[7-Zip]:http://www.7-zip.org/ -[lzip]:http://www.nongnu.org/lzip/ +[7-Zip]:https://www.7-zip.org/ +[lzip]:https://www.nongnu.org/lzip/ [Brotli]:https://github.com/google/brotli/ [LZ4]:https://github.com/lz4/lz4/ [LZ5]:https://github.com/inikep/lz5/ diff --git a/cleanup.cmd b/cleanup.cmd index be556047..6b9d9487 100644 --- a/cleanup.cmd +++ b/cleanup.cmd @@ -1,6 +1,6 @@ @echo off -FOR /R .\ %%d IN (AMD64 IA64 O) DO rd /S /Q %%d 2>NUL +FOR /R .\ %%d IN (arm x64 O) DO rd /S /Q %%d 2>NUL del "CPP\7zip\*.7z" 2>NUL del "CPP\7zip\*.exe" 2>NUL